refactor(hive-c0re): group src-root files into submodules
stores/ (sqlite-backed host stores + db helper), stats/, agent_config/, workers/ — pure git-mv moves; crate-root re-exports keep every crate::<module> path compiling. flake_check stays at root (synchronous approval-flow validation, not a background worker)
This commit is contained in:
parent
b489454dc2
commit
0e4b5a1120
29 changed files with 68 additions and 24 deletions
|
|
@ -413,7 +413,7 @@ tool group can therefore edit, commit, and submit changes for any of
|
||||||
its direct children directly inside its container at `/agents/<child>/config/`.
|
its direct children directly inside its container at `/agents/<child>/config/`.
|
||||||
|
|
||||||
Agents holding the `can_manage_top_level_agents` topology role
|
Agents holding the `can_manage_top_level_agents` topology role
|
||||||
(defined as `ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS` in `hive-c0re/src/topology.rs`)
|
(defined as `ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS` in `hive-c0re/src/agent_config/topology.rs`)
|
||||||
get additional host-side bind mounts via `set_nspawn_flags`:
|
get additional host-side bind mounts via `set_nspawn_flags`:
|
||||||
|
|
||||||
- `/var/lib/hyperhive/agents/` → `/agents/` (RW) — all top-level
|
- `/var/lib/hyperhive/agents/` → `/agents/` (RW) — all top-level
|
||||||
|
|
|
||||||
|
|
@ -109,7 +109,7 @@ Notable collapses:
|
||||||
### Desired-state (spec vs status)
|
### Desired-state (spec vs status)
|
||||||
|
|
||||||
Per-agent power *intent* — `wanted: Up | Offline` — is durable as the
|
Per-agent power *intent* — `wanted: Up | Offline` — is durable as the
|
||||||
`agent_power` table in the coordinator DB (`hive-c0re/src/power.rs`).
|
`agent_power` table in the coordinator DB (`hive-c0re/src/stores/power.rs`).
|
||||||
`container_view` remains the observed *status*; `Reconcile` nodes converge the
|
`container_view` remains the observed *status*; `Reconcile` nodes converge the
|
||||||
two. Setting `wanted` is never a queued node: the submit layer
|
two. Setting `wanted` is never a queued node: the submit layer
|
||||||
(`job_queue/submit.rs`) writes the row synchronously, then submits the DAG
|
(`job_queue/submit.rs`) writes the row synchronously, then submits the DAG
|
||||||
|
|
|
||||||
|
|
@ -47,7 +47,7 @@ header/targets split, and the per-agent power-intent registry:
|
||||||
- `agent_power` — one tiny row per agent: `agent PK / wanted (up |
|
- `agent_power` — one tiny row per agent: `agent PK / wanted (up |
|
||||||
offline) / updated_at` — the durable power *intent* behind the job
|
offline) / updated_at` — the durable power *intent* behind the job
|
||||||
queue's desired-state reconciliation
|
queue's desired-state reconciliation
|
||||||
(`docs/coordinator.md::Job queue`; owner: `hive-c0re/src/power.rs`).
|
(`docs/coordinator.md::Job queue`; owner: `hive-c0re/src/stores/power.rs`).
|
||||||
Written synchronously by every operator/agent power action
|
Written synchronously by every operator/agent power action
|
||||||
(dashboard start/stop, `hivectl stop`, the MCP kill/start tools,
|
(dashboard start/stop, `hivectl stop`, the MCP kill/start tools,
|
||||||
spawn approval); read by `Reconcile` nodes and the boot reconcile.
|
spawn approval); read by `Reconcile` nodes and the boot reconcile.
|
||||||
|
|
|
||||||
95
hive-c0re/src/agent_config/capabilities.rs
Normal file
95
hive-c0re/src/agent_config/capabilities.rs
Normal file
|
|
@ -0,0 +1,95 @@
|
||||||
|
//! Per-agent capability configuration. Stored at
|
||||||
|
//! `/var/lib/hyperhive/meta/capabilities.json` alongside `topology.json`
|
||||||
|
//! and `tool-groups.json`.
|
||||||
|
//!
|
||||||
|
//! Format: a JSON object mapping agent name to an array of
|
||||||
|
//! `hive_sh4re::Capability` `snake_case` strings:
|
||||||
|
//!
|
||||||
|
//! ```json
|
||||||
|
//! {
|
||||||
|
//! "atlas": ["read_host_journal"],
|
||||||
|
//! "ruth": ["query_agent_state"]
|
||||||
|
//! }
|
||||||
|
//! ```
|
||||||
|
//!
|
||||||
|
//! An absent entry (or an absent file) means "no extra capabilities".
|
||||||
|
//! `render_flake` in `meta.rs` reads this file and injects
|
||||||
|
//! `HIVE_CAPABILITIES` into each agent's systemd service env; absent
|
||||||
|
//! entries emit no env var so agents without capabilities don't trigger
|
||||||
|
//! a spurious rebuild.
|
||||||
|
//!
|
||||||
|
//! Write path: `set_caps` is called from the dashboard action handler
|
||||||
|
//! that the operator uses to grant/revoke capabilities per agent.
|
||||||
|
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
const CAPABILITIES_FILE: &str = "capabilities.json";
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
|
pub fn capabilities_path() -> PathBuf {
|
||||||
|
crate::meta::meta_dir().join(CAPABILITIES_FILE)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read the per-agent capability map. Returns an empty map when the
|
||||||
|
/// file is absent or unparsable — callers treat a missing entry as
|
||||||
|
/// "no extra capabilities".
|
||||||
|
#[must_use]
|
||||||
|
pub fn read() -> BTreeMap<String, Vec<String>> {
|
||||||
|
let path = capabilities_path();
|
||||||
|
let Ok(raw) = std::fs::read_to_string(&path) else {
|
||||||
|
return BTreeMap::new();
|
||||||
|
};
|
||||||
|
serde_json::from_str(&raw).unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Look up the configured capabilities for one agent. Returns an empty
|
||||||
|
/// vec when the agent has no entry.
|
||||||
|
#[must_use]
|
||||||
|
pub fn caps_for(name: &str) -> Vec<String> {
|
||||||
|
read().get(name).cloned().unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check whether an agent holds a specific capability.
|
||||||
|
#[must_use]
|
||||||
|
pub fn has_cap(name: &str, cap: hive_sh4re::Capability) -> bool {
|
||||||
|
caps_for(name)
|
||||||
|
.iter()
|
||||||
|
.any(|s| s.eq_ignore_ascii_case(cap.as_str()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Persist the full capability map. Sorted JSON output keeps diffs
|
||||||
|
/// minimal. Best-effort — returns `io::Error` so callers decide
|
||||||
|
/// whether to abort or log.
|
||||||
|
pub fn write(map: &BTreeMap<String, Vec<String>>) -> std::io::Result<()> {
|
||||||
|
let path = capabilities_path();
|
||||||
|
if let Some(parent) = path.parent() {
|
||||||
|
std::fs::create_dir_all(parent)?;
|
||||||
|
}
|
||||||
|
let text = serde_json::to_string_pretty(map)
|
||||||
|
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
|
||||||
|
std::fs::write(&path, format!("{text}\n"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Set the capabilities for one agent and persist the map. An empty
|
||||||
|
/// `caps` vec removes the entry (agent has no capabilities).
|
||||||
|
pub fn set_caps(name: &str, caps: &[String]) -> std::io::Result<()> {
|
||||||
|
let mut current = read();
|
||||||
|
if caps.is_empty() {
|
||||||
|
current.remove(name);
|
||||||
|
} else {
|
||||||
|
current.insert(name.to_owned(), caps.to_vec());
|
||||||
|
}
|
||||||
|
write(¤t)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Remove an agent from the capability map entirely. Called by
|
||||||
|
/// `meta::sync_agents` when an agent is deprovisioned so stale entries
|
||||||
|
/// don't accumulate. No-op if the agent has no entry.
|
||||||
|
pub fn remove_agent(name: &str) -> std::io::Result<()> {
|
||||||
|
let mut current = read();
|
||||||
|
if current.remove(name).is_some() {
|
||||||
|
write(¤t)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
176
hive-c0re/src/agent_config/limits.rs
Normal file
176
hive-c0re/src/agent_config/limits.rs
Normal file
|
|
@ -0,0 +1,176 @@
|
||||||
|
//! Wire-protocol size limits shared across the agent + manager
|
||||||
|
//! sockets. Caps on inline message bodies stop a single chatty agent
|
||||||
|
//! (or a misbehaving extra-MCP server) from flooding the broker
|
||||||
|
//! sqlite with megabyte-sized rows that then bloat every recipient's
|
||||||
|
//! wake-prompt context. Anything genuinely larger should be written
|
||||||
|
//! to a state file and the path sent as the body.
|
||||||
|
//!
|
||||||
|
//! Reminders get a separate auto-file escape hatch (see
|
||||||
|
//! `socket_server::handle_remind`) so callers don't have to think
|
||||||
|
//! about it — oversized reminder bodies get persisted to disk
|
||||||
|
//! transparently and the inbox sees a pointer.
|
||||||
|
|
||||||
|
/// Per-message body cap. Applies to `send`, `ask` question text,
|
||||||
|
/// `answer` body, and the stored inline form of a reminder. 4 KiB
|
||||||
|
/// catches the bulk of conversational overflow (status reports,
|
||||||
|
/// bullet-list summaries, short proposals) while staying small
|
||||||
|
/// enough that a backed-up inbox of ~10 unread messages only adds
|
||||||
|
/// ~40 KiB to the recipient's wake-prompt context. Genuinely
|
||||||
|
/// long-form artifacts (audit reports, full diffs, transcripts)
|
||||||
|
/// still belong in a state file — the error message on overflow
|
||||||
|
/// points callers at that escape hatch.
|
||||||
|
pub const MESSAGE_MAX_BYTES: usize = 4096;
|
||||||
|
|
||||||
|
/// Validate that `body` fits under [`MESSAGE_MAX_BYTES`]. Returns a
|
||||||
|
/// caller-ready error string (caller wraps in
|
||||||
|
/// `AgentResponse::Err`/`ManagerResponse::Err`) on failure.
|
||||||
|
///
|
||||||
|
/// `label` shows up in the error message verbatim — pass a short
|
||||||
|
/// noun like `"send"`, `"question"`, `"broadcast"` so the model can
|
||||||
|
/// tell which call got rejected.
|
||||||
|
pub fn check_size(label: &str, body: &str) -> Result<(), String> {
|
||||||
|
if body.len() > MESSAGE_MAX_BYTES {
|
||||||
|
Err(format!(
|
||||||
|
"{label} body too long ({} bytes, max {MESSAGE_MAX_BYTES}); write the \
|
||||||
|
payload to a file under your `/agents/<you>/state/` dir and send the \
|
||||||
|
path as the body instead",
|
||||||
|
body.len()
|
||||||
|
))
|
||||||
|
} else {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Per-status soft cap. `set_status` renders as a short chip on the
|
||||||
|
/// dashboard agent card — the front-end truncates long strings to
|
||||||
|
/// keep the row layout intact, so a multi-paragraph "session report"
|
||||||
|
/// is wasted bytes that just bloat the rescan emit + container view
|
||||||
|
/// payload. Cap at 200 chars to fit the chip plus a little
|
||||||
|
/// descriptive padding without forcing the operator to read a
|
||||||
|
/// scrolling chunk.
|
||||||
|
/// NOTE: `hive-ag3nt/src/mcp.rs::write_status_file` mirrors this constant
|
||||||
|
/// client-side so invalid text is caught before the file is written.
|
||||||
|
/// Keep in sync if this value changes.
|
||||||
|
pub const STATUS_MAX_CHARS: usize = 200;
|
||||||
|
|
||||||
|
/// Validate a `set_status` payload. Single-line + bounded so
|
||||||
|
/// callers can't dump multi-paragraph session reports into the
|
||||||
|
/// dashboard chip. Whitespace trim is done by the caller before the
|
||||||
|
/// store-to-disk step — we run validation on the trimmed form so
|
||||||
|
/// surrounding whitespace doesn't push a borderline-legal status
|
||||||
|
/// past the cap.
|
||||||
|
///
|
||||||
|
/// Empty / all-whitespace input is accepted: the call site treats
|
||||||
|
/// that as "clear the status" and removes the on-disk sentinel. Tests
|
||||||
|
/// + caller cover both directions.
|
||||||
|
///
|
||||||
|
/// Returns a caller-ready error string suitable for surfacing in the
|
||||||
|
/// `*Response::Err` shape.
|
||||||
|
pub fn check_status_text(text: &str) -> Result<(), String> {
|
||||||
|
let trimmed = text.trim();
|
||||||
|
if trimmed.is_empty() {
|
||||||
|
// Empty = clear-status sentinel; nothing to validate.
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
// Newline / carriage-return: status is a single-line chip on the
|
||||||
|
// dashboard. Multi-line session reports should go to a state file.
|
||||||
|
if trimmed.contains('\n') || trimmed.contains('\r') {
|
||||||
|
return Err(
|
||||||
|
"set_status text must be a single line — write multi-line context to \
|
||||||
|
a file under your `/agents/<you>/state/` dir and reference that path \
|
||||||
|
from the chip instead"
|
||||||
|
.to_owned(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let len = trimmed.chars().count();
|
||||||
|
if len > STATUS_MAX_CHARS {
|
||||||
|
return Err(format!(
|
||||||
|
"set_status text too long ({len} chars, max {STATUS_MAX_CHARS}); the \
|
||||||
|
dashboard chip truncates anything longer, so trim to a short summary \
|
||||||
|
and write the detail to `/agents/<you>/state/<file>` instead"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn accepts_short_body() {
|
||||||
|
assert!(check_size("send", "hello").is_ok());
|
||||||
|
assert!(check_size("send", &"x".repeat(MESSAGE_MAX_BYTES)).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rejects_oversize_body() {
|
||||||
|
let err = check_size("send", &"x".repeat(MESSAGE_MAX_BYTES + 1)).unwrap_err();
|
||||||
|
assert!(err.contains("send body too long"));
|
||||||
|
assert!(err.contains(&format!("max {MESSAGE_MAX_BYTES}")));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn label_threads_through() {
|
||||||
|
let err = check_size("question", &"x".repeat(MESSAGE_MAX_BYTES + 1)).unwrap_err();
|
||||||
|
assert!(err.starts_with("question body too long"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn check_status_accepts_short_single_line() {
|
||||||
|
assert!(check_status_text("idle").is_ok());
|
||||||
|
assert!(check_status_text("processing matrix messages").is_ok());
|
||||||
|
// Boundary: exactly STATUS_MAX_CHARS chars trimmed is still
|
||||||
|
// accepted; one more rejects.
|
||||||
|
let max = "a".repeat(STATUS_MAX_CHARS);
|
||||||
|
assert!(check_status_text(&max).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn check_status_accepts_empty_and_whitespace() {
|
||||||
|
// Empty + whitespace-only are the "clear status" sentinel and
|
||||||
|
// bypass the rest of the checks.
|
||||||
|
assert!(check_status_text("").is_ok());
|
||||||
|
assert!(check_status_text(" ").is_ok());
|
||||||
|
assert!(check_status_text("\n\t ").is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn check_status_rejects_multi_line() {
|
||||||
|
let err = check_status_text("line one\nline two").unwrap_err();
|
||||||
|
assert!(err.contains("single line"), "err = {err}");
|
||||||
|
// Carriage return alone also rejects (windows linebreak / CR-only).
|
||||||
|
assert!(check_status_text("a\rb").is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn check_status_rejects_oversize() {
|
||||||
|
let too_long = "a".repeat(STATUS_MAX_CHARS + 1);
|
||||||
|
let err = check_status_text(&too_long).unwrap_err();
|
||||||
|
assert!(err.contains("too long"), "err = {err}");
|
||||||
|
assert!(err.contains(&format!("max {STATUS_MAX_CHARS}")));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn check_status_counts_chars_not_bytes() {
|
||||||
|
// Multi-byte chars (emoji, accented letters) count once each
|
||||||
|
// per char — chars().count() not byte len. STATUS_MAX_CHARS
|
||||||
|
// worth of 4-byte chars is still legal.
|
||||||
|
let emoji = "💜".repeat(STATUS_MAX_CHARS);
|
||||||
|
assert!(
|
||||||
|
check_status_text(&emoji).is_ok(),
|
||||||
|
"{STATUS_MAX_CHARS} emoji should fit"
|
||||||
|
);
|
||||||
|
let too_many = "💜".repeat(STATUS_MAX_CHARS + 1);
|
||||||
|
assert!(check_status_text(&too_many).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn check_status_validates_post_trim() {
|
||||||
|
// Leading/trailing whitespace is trimmed before the length
|
||||||
|
// check — a borderline-legal payload with spaces around it
|
||||||
|
// still passes.
|
||||||
|
let padded = format!(" {} ", "a".repeat(STATUS_MAX_CHARS));
|
||||||
|
assert!(check_status_text(&padded).is_ok());
|
||||||
|
}
|
||||||
|
}
|
||||||
10
hive-c0re/src/agent_config/mod.rs
Normal file
10
hive-c0re/src/agent_config/mod.rs
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
//! Per-agent configuration registries: tool groups, capabilities,
|
||||||
|
//! topology (all JSON files under `/var/lib/hyperhive/meta/`) and the
|
||||||
|
//! shared wire-protocol size limits. Each submodule is re-exported at
|
||||||
|
//! the crate root, so `crate::topology::…` etc. keep working
|
||||||
|
//! unchanged.
|
||||||
|
|
||||||
|
pub mod capabilities;
|
||||||
|
pub mod limits;
|
||||||
|
pub mod tool_groups;
|
||||||
|
pub mod topology;
|
||||||
120
hive-c0re/src/agent_config/tool_groups.rs
Normal file
120
hive-c0re/src/agent_config/tool_groups.rs
Normal file
|
|
@ -0,0 +1,120 @@
|
||||||
|
//! Per-agent tool-group configuration. Stored at
|
||||||
|
//! `/var/lib/hyperhive/meta/tool-groups.json` alongside `topology.json`
|
||||||
|
//! and the meta `flake.nix`.
|
||||||
|
//!
|
||||||
|
//! Format: a JSON object mapping agent name to an array of
|
||||||
|
//! `hive_sh4re::ToolGroup` `snake_case` strings:
|
||||||
|
//!
|
||||||
|
//! ```json
|
||||||
|
//! {
|
||||||
|
//! "alice": ["messaging", "meta", "inbox", "lifecycle"],
|
||||||
|
//! "bob": ["messaging", "meta", "inbox"]
|
||||||
|
//! }
|
||||||
|
//! ```
|
||||||
|
//!
|
||||||
|
//! An absent entry (or an absent file) means "use the harness default"
|
||||||
|
//! (`AGENT_DEFAULT`: `messaging + meta + inbox + execution`). `render_flake`
|
||||||
|
//! in `meta.rs` reads this file and injects `HIVE_TOOL_GROUPS` into each
|
||||||
|
//! agent's systemd service env; agents with no entry get no env var and the
|
||||||
|
//! harness falls back to `AGENT_DEFAULT`.
|
||||||
|
//!
|
||||||
|
//! Write path: `set_groups` is called from the dashboard action handler
|
||||||
|
//! that the operator uses to grant/revoke tool groups per agent.
|
||||||
|
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
use anyhow::Context as _;
|
||||||
|
|
||||||
|
const TOOL_GROUPS_FILE: &str = "tool-groups.json";
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
|
pub fn tool_groups_path() -> PathBuf {
|
||||||
|
crate::meta::meta_dir().join(TOOL_GROUPS_FILE)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read the per-agent tool-group map. Returns an empty map when the
|
||||||
|
/// file is absent or unparsable — callers treat a missing entry as
|
||||||
|
/// "use role default".
|
||||||
|
#[must_use]
|
||||||
|
pub fn read() -> BTreeMap<String, Vec<String>> {
|
||||||
|
let path = tool_groups_path();
|
||||||
|
let Ok(raw) = std::fs::read_to_string(&path) else {
|
||||||
|
return BTreeMap::new();
|
||||||
|
};
|
||||||
|
serde_json::from_str(&raw).unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Look up the configured tool groups for one agent. Returns an empty
|
||||||
|
/// vec when the agent has no entry — callers should treat this as
|
||||||
|
/// "use the harness role default."
|
||||||
|
#[must_use]
|
||||||
|
pub fn groups_for(name: &str) -> Vec<String> {
|
||||||
|
read().get(name).cloned().unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Persist the full tool-groups map. Sorted JSON output keeps diffs
|
||||||
|
/// minimal. Use `set_groups` (or `remove_agent`) from outside this
|
||||||
|
/// module — they go through the validated write path.
|
||||||
|
fn write(map: &BTreeMap<String, Vec<String>>) -> std::io::Result<()> {
|
||||||
|
let path = tool_groups_path();
|
||||||
|
if let Some(parent) = path.parent() {
|
||||||
|
std::fs::create_dir_all(parent)?;
|
||||||
|
}
|
||||||
|
let text = serde_json::to_string_pretty(map)
|
||||||
|
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
|
||||||
|
std::fs::write(&path, format!("{text}\n"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Validate a slice of group name strings against `ToolGroup::ALL`.
|
||||||
|
/// Returns `Ok(())` when all names are known, or `Err` listing the
|
||||||
|
/// unrecognised names so callers can surface a useful error message.
|
||||||
|
pub fn validate_groups(groups: &[String]) -> anyhow::Result<()> {
|
||||||
|
let valid: std::collections::BTreeSet<&str> = hive_sh4re::ToolGroup::ALL
|
||||||
|
.iter()
|
||||||
|
.map(|g| g.as_str())
|
||||||
|
.collect();
|
||||||
|
let unknown: Vec<&str> = groups
|
||||||
|
.iter()
|
||||||
|
.map(String::as_str)
|
||||||
|
.filter(|s| !valid.contains(s))
|
||||||
|
.collect();
|
||||||
|
if unknown.is_empty() {
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
anyhow::bail!(
|
||||||
|
"unknown tool group(s): {}; valid names are: {}",
|
||||||
|
unknown.join(", "),
|
||||||
|
hive_sh4re::ToolGroup::ALL
|
||||||
|
.iter()
|
||||||
|
.map(|g| g.as_str())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", ")
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Set the tool groups for one agent and persist the map. An empty
|
||||||
|
/// `groups` vec removes the entry (agent reverts to role default).
|
||||||
|
/// Returns an error if any name is not in `ToolGroup::ALL`.
|
||||||
|
pub fn set_groups(name: &str, groups: &[String]) -> anyhow::Result<()> {
|
||||||
|
if !groups.is_empty() {
|
||||||
|
validate_groups(groups)?;
|
||||||
|
}
|
||||||
|
let mut current = read();
|
||||||
|
if groups.is_empty() {
|
||||||
|
current.remove(name);
|
||||||
|
} else {
|
||||||
|
current.insert(name.to_owned(), groups.to_vec());
|
||||||
|
}
|
||||||
|
write(¤t).with_context(|| format!("write tool-groups for {name}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Drop the entry for an agent that is being destroyed. Idempotent.
|
||||||
|
pub fn remove_agent(name: &str) -> std::io::Result<()> {
|
||||||
|
let mut current = read();
|
||||||
|
if current.remove(name).is_some() {
|
||||||
|
write(¤t)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
912
hive-c0re/src/agent_config/topology.rs
Normal file
912
hive-c0re/src/agent_config/topology.rs
Normal file
|
|
@ -0,0 +1,912 @@
|
||||||
|
//! Agent topology storage — single source of truth for parent/child
|
||||||
|
//! relations in the hive. Persisted as a flat JSON map of `name →
|
||||||
|
//! parent name | null` at `/var/lib/hyperhive/meta/topology.json`,
|
||||||
|
//! alongside the meta `flake.nix`, so topology changes thread through
|
||||||
|
//! the same git commit log as deploys.
|
||||||
|
//!
|
||||||
|
//! Agent roles are stored alongside in `roles.json` as a flat map of
|
||||||
|
//! `name → [role, ...]`. Roles gate additional bind-mount grants; see
|
||||||
|
//! `lifecycle::set_nspawn_flags` for the consumer.
|
||||||
|
//!
|
||||||
|
//! Format, rationale, read/reconcile/inject/surface flow, and target
|
||||||
|
//! enforcement semantics: `docs/agent-hierarchy.md::Current state`.
|
||||||
|
//! `<parent>` sentinel resolution (delivered by [`resolve_recipient`]):
|
||||||
|
//! `docs/conventions.md::Recipient sentinels`.
|
||||||
|
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
const TOPOLOGY_FILE: &str = "topology.json";
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
|
pub fn topology_path() -> PathBuf {
|
||||||
|
crate::meta::meta_dir().join(TOPOLOGY_FILE)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Snapshot of the topology map. Read on every `container_view::build_all`
|
||||||
|
/// and every `render_flake` call. The file is small (one line per agent),
|
||||||
|
/// so we re-read rather than caching — keeps the source of truth on disk.
|
||||||
|
///
|
||||||
|
/// Returns an empty map when the file is absent or unparsable; callers
|
||||||
|
/// treat that as "no recorded parents", which falls back to every agent
|
||||||
|
/// being root-level. Safe degradation for fresh installs that haven't
|
||||||
|
/// run through `meta::sync_agents` yet.
|
||||||
|
#[must_use]
|
||||||
|
pub fn read() -> BTreeMap<String, Option<String>> {
|
||||||
|
let path = topology_path();
|
||||||
|
let Ok(raw) = std::fs::read_to_string(&path) else {
|
||||||
|
return BTreeMap::new();
|
||||||
|
};
|
||||||
|
serde_json::from_str(&raw).unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Look up one agent's parent. Returns `None` when the agent is root
|
||||||
|
/// or absent from the file. Cheap convenience over `read()` for
|
||||||
|
/// callers that want a single entry.
|
||||||
|
#[must_use]
|
||||||
|
pub fn parent_of(name: &str) -> Option<String> {
|
||||||
|
read().get(name).cloned().flatten()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return the direct children of `name` — agents whose `topology.json`
|
||||||
|
/// entry has `name` as their parent. Reads the map once and scans all
|
||||||
|
/// entries; cheap enough for the fan-out path (one disk read per send
|
||||||
|
/// to `<children>`).
|
||||||
|
#[must_use]
|
||||||
|
pub fn children_of(name: &str) -> Vec<String> {
|
||||||
|
children_of_in(&read(), name)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pure form of [`children_of`] for unit tests.
|
||||||
|
#[must_use]
|
||||||
|
pub fn children_of_in(topo: &BTreeMap<String, Option<String>>, name: &str) -> Vec<String> {
|
||||||
|
topo.iter()
|
||||||
|
.filter_map(|(agent, parent)| {
|
||||||
|
if parent.as_deref() == Some(name) {
|
||||||
|
Some(agent.clone())
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return every agent that has no parent in the topology. These are the
|
||||||
|
/// "top-level" agents a `can_manage_top_level_agents` role holder is
|
||||||
|
/// granted access to. No agent name is hardcoded — the set is derived
|
||||||
|
/// purely from topology structure.
|
||||||
|
///
|
||||||
|
/// In normal operation this is just the manager, but any agent the
|
||||||
|
/// operator explicitly places outside the hierarchy is also included.
|
||||||
|
#[must_use]
|
||||||
|
pub fn top_level_agents() -> Vec<String> {
|
||||||
|
top_level_agents_in(&read())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pure form of [`top_level_agents`] for unit tests.
|
||||||
|
#[must_use]
|
||||||
|
pub fn top_level_agents_in(topo: &BTreeMap<String, Option<String>>) -> Vec<String> {
|
||||||
|
topo.iter()
|
||||||
|
.filter_map(|(name, parent)| {
|
||||||
|
if parent.is_none() {
|
||||||
|
Some(name.clone())
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolve a magic recipient sentinel (currently just
|
||||||
|
/// [`hive_sh4re::PARENT_RECIPIENT`]) to a real broker recipient at
|
||||||
|
/// send time. Returns an owned `String` so callers can plug it
|
||||||
|
/// straight into [`crate::broker::Broker::send`] without
|
||||||
|
/// borrow-juggling around the temporary lookup.
|
||||||
|
///
|
||||||
|
/// Rules + rationale: `docs/conventions.md::Recipient sentinels`.
|
||||||
|
/// Fast path: ordinary recipient names short-circuit before any
|
||||||
|
/// disk read — only `<parent>` triggers `read()` on `topology.json`.
|
||||||
|
#[must_use]
|
||||||
|
pub fn resolve_recipient(sender: &str, to: &str) -> String {
|
||||||
|
// Early exit: only sentinel recipients need topology lookup. This
|
||||||
|
// keeps the cost of a normal `send` at one string comparison.
|
||||||
|
if to != hive_sh4re::PARENT_RECIPIENT {
|
||||||
|
return to.to_owned();
|
||||||
|
}
|
||||||
|
resolve_recipient_in(&read(), sender, to)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pure form of [`resolve_recipient`] taking the topology map
|
||||||
|
/// explicitly. Split out so unit tests can exercise the sentinel
|
||||||
|
/// rules without writing a `topology.json` to disk.
|
||||||
|
#[must_use]
|
||||||
|
pub fn resolve_recipient_in(
|
||||||
|
topo: &BTreeMap<String, Option<String>>,
|
||||||
|
sender: &str,
|
||||||
|
to: &str,
|
||||||
|
) -> String {
|
||||||
|
if to == hive_sh4re::PARENT_RECIPIENT {
|
||||||
|
topo.get(sender)
|
||||||
|
.cloned()
|
||||||
|
.flatten()
|
||||||
|
.unwrap_or_else(|| hive_sh4re::OPERATOR_RECIPIENT.to_owned())
|
||||||
|
} else {
|
||||||
|
to.to_owned()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// True when `candidate` is `ancestor` or any descendant of
|
||||||
|
/// `ancestor` per the current topology. Walks parents from
|
||||||
|
/// `candidate` upward; the walk terminates at root or on a cycle
|
||||||
|
/// (cycle defence: bounded to 32 hops, more than any plausible
|
||||||
|
/// hive depth). Used by the cancel-authorization check in
|
||||||
|
/// `socket_server::handle_cancel_schedule` to enforce
|
||||||
|
/// "managers can cancel anything their subtree owns."
|
||||||
|
#[must_use]
|
||||||
|
pub fn is_descendant_of(candidate: &str, ancestor: &str) -> bool {
|
||||||
|
if candidate == ancestor {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
let topo = read();
|
||||||
|
let mut cur = candidate.to_owned();
|
||||||
|
for _ in 0..32 {
|
||||||
|
let Some(parent) = topo.get(&cur).cloned().flatten() else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
if parent == ancestor {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
cur = parent;
|
||||||
|
}
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Persist the topology map. Sorted JSON output (`BTreeMap` is sorted by
|
||||||
|
/// key) keeps git diffs minimal across re-writes. Best-effort —
|
||||||
|
/// returns an `io::Error` so callers can decide whether a failure
|
||||||
|
/// should abort their op (`sync_agents`, `RequestSetParent`) or just log.
|
||||||
|
pub fn write(topology: &BTreeMap<String, Option<String>>) -> std::io::Result<()> {
|
||||||
|
let path = topology_path();
|
||||||
|
if let Some(parent) = path.parent() {
|
||||||
|
std::fs::create_dir_all(parent)?;
|
||||||
|
}
|
||||||
|
let text = serde_json::to_string_pretty(topology)
|
||||||
|
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
|
||||||
|
std::fs::write(&path, format!("{text}\n"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Compute the default topology for a fresh install: every agent is a
|
||||||
|
/// root (parent = null). There is no structural "manager" — agents
|
||||||
|
/// arrange themselves via explicit parent edges (an agent-requested
|
||||||
|
/// sub-agent gets a requester-as-parent edge at `init_config`; the
|
||||||
|
/// operator reparents via the dashboard / `RequestSetParent` API).
|
||||||
|
/// Used by `meta::sync_agents` on first call to seed `topology.json`.
|
||||||
|
///
|
||||||
|
/// As soon as an explicit write lands (dashboard / `RequestSetParent`
|
||||||
|
/// API), this seeding stops touching pre-existing entries —
|
||||||
|
/// `sync_agents` only adds rows for newly-spawned agents against
|
||||||
|
/// whatever the operator has configured.
|
||||||
|
#[must_use]
|
||||||
|
#[allow(
|
||||||
|
dead_code,
|
||||||
|
reason = "kept for the dashboard / RequestSetParent write API; \
|
||||||
|
`sync_agents` does its own seeding today"
|
||||||
|
)]
|
||||||
|
pub fn default_seed(agent_names: &[String]) -> BTreeMap<String, Option<String>> {
|
||||||
|
let mut out = BTreeMap::new();
|
||||||
|
for name in agent_names {
|
||||||
|
out.insert(name.clone(), None);
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pure validation + apply for [`set_parent`]. Splits off so tests
|
||||||
|
/// can exercise the rules (cycle / unknown) on an in-memory
|
||||||
|
/// `BTreeMap` without touching the on-disk `topology.json`. Returns
|
||||||
|
/// either the post-move map (caller writes it back) or a
|
||||||
|
/// user-readable error string.
|
||||||
|
///
|
||||||
|
/// The manager is reparentable like any other agent — its special
|
||||||
|
/// powers come from the privileged MCP socket, not its tree
|
||||||
|
/// position. The cycle walk below covers "moving X under its own
|
||||||
|
/// descendant" for the manager as much as any other agent.
|
||||||
|
/// `docs/agent-hierarchy.md::Current state` has the rationale.
|
||||||
|
pub fn apply_set_parent(
|
||||||
|
topo: &BTreeMap<String, Option<String>>,
|
||||||
|
child: &str,
|
||||||
|
new_parent: Option<&str>,
|
||||||
|
) -> Result<BTreeMap<String, Option<String>>, String> {
|
||||||
|
if !topo.contains_key(child) {
|
||||||
|
return Err(format!("unknown agent: {child}"));
|
||||||
|
}
|
||||||
|
if let Some(p) = new_parent {
|
||||||
|
if !topo.contains_key(p) {
|
||||||
|
return Err(format!("unknown parent: {p}"));
|
||||||
|
}
|
||||||
|
if p == child {
|
||||||
|
return Err("an agent cannot be its own parent".to_owned());
|
||||||
|
}
|
||||||
|
// Cycle check: walk `p`'s ancestors in the EXISTING map. If
|
||||||
|
// we hit `child`, then making `child`'s parent = `p` would
|
||||||
|
// close the loop (child → … → p → child).
|
||||||
|
let mut cur = p.to_owned();
|
||||||
|
for _ in 0..32 {
|
||||||
|
if cur == child {
|
||||||
|
return Err(format!(
|
||||||
|
"cycle: {p} is in {child}'s subtree (would create a loop)"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let Some(next) = topo.get(&cur).cloned().flatten() else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
|
cur = next;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut next = topo.clone();
|
||||||
|
next.insert(child.to_owned(), new_parent.map(str::to_owned));
|
||||||
|
Ok(next)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Operator-driven parent move. Set `child`'s parent to `new_parent`
|
||||||
|
/// (or `None` to promote to root). See [`apply_set_parent`] for the
|
||||||
|
/// validation rules. The operator-set parent sticks across
|
||||||
|
/// `reconcile()` calls (which preserves existing entries).
|
||||||
|
///
|
||||||
|
/// No bind-mount / container churn today — the hierarchy is
|
||||||
|
/// currently logical-only. Once sub-manager bind mounts land, the
|
||||||
|
/// caller adds an umount-old / mount-new / restart-cascade step on
|
||||||
|
/// top.
|
||||||
|
pub fn set_parent(child: &str, new_parent: Option<&str>) -> Result<(), String> {
|
||||||
|
let current = read();
|
||||||
|
// Idempotent no-op fast path: skip the disk write when nothing
|
||||||
|
// changes. apply_set_parent still runs to surface validation
|
||||||
|
// errors (e.g. unknown child) so the caller gets a real signal.
|
||||||
|
let next = apply_set_parent(¤t, child, new_parent)?;
|
||||||
|
if next == current {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
write(&next).map_err(|e| format!("write topology.json: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Declare a brand-new agent's parent edge before the agent exists in
|
||||||
|
/// the container set. Unlike [`set_parent`] (which reparents an entry
|
||||||
|
/// that must already be present), this inserts a fresh `child -> parent`
|
||||||
|
/// row. Used by the `InitConfig` approval to place a just-scaffolded
|
||||||
|
/// sub-agent under its requesting parent, so the edge is in place
|
||||||
|
/// before the first apply-commit spawns the container (and before
|
||||||
|
/// [`reconcile`] would otherwise default it to the manager).
|
||||||
|
///
|
||||||
|
/// Idempotent when the edge already exists. Refuses to clobber an entry
|
||||||
|
/// whose parent differs (one agent can't steal another's child) and
|
||||||
|
/// validates that `parent` is itself a known agent.
|
||||||
|
pub fn add_child(child: &str, parent: &str) -> Result<(), String> {
|
||||||
|
let current = read();
|
||||||
|
match apply_add_child(¤t, child, parent)? {
|
||||||
|
Some(next) => write(&next).map_err(|e| format!("write topology.json: {e}")),
|
||||||
|
None => Ok(()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pure form of [`add_child`] for unit tests. Returns the post-insert
|
||||||
|
/// map (caller writes it back), `None` for an idempotent no-op (edge
|
||||||
|
/// already present), or an error string (unknown parent / name owned by
|
||||||
|
/// a different parent).
|
||||||
|
pub fn apply_add_child(
|
||||||
|
topo: &BTreeMap<String, Option<String>>,
|
||||||
|
child: &str,
|
||||||
|
parent: &str,
|
||||||
|
) -> Result<Option<BTreeMap<String, Option<String>>>, String> {
|
||||||
|
if !topo.contains_key(parent) {
|
||||||
|
return Err(format!("unknown parent: {parent}"));
|
||||||
|
}
|
||||||
|
match topo.get(child) {
|
||||||
|
Some(Some(p)) if p == parent => return Ok(None),
|
||||||
|
Some(existing) => {
|
||||||
|
return Err(format!(
|
||||||
|
"agent {child} already exists in topology (parent: {existing:?}); \
|
||||||
|
refusing to reparent via add_child"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
None => {}
|
||||||
|
}
|
||||||
|
let mut next = topo.clone();
|
||||||
|
next.insert(child.to_owned(), Some(parent.to_owned()));
|
||||||
|
Ok(Some(next))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reconcile `topology.json` against the current agent set. Adds an
|
||||||
|
/// entry (default: parent = null — a new agent with no declared parent
|
||||||
|
/// is its own root) for any agent missing from the file; removes
|
||||||
|
/// entries for agents no longer
|
||||||
|
/// present. Existing entries are preserved as-is — operator/manager
|
||||||
|
/// choices stick across regenerations. Returns true when the file
|
||||||
|
/// changed and should be re-committed by the caller.
|
||||||
|
///
|
||||||
|
/// `pending` lists agents that have an operator-approved proposed config
|
||||||
|
/// repo but no container yet (init'd, not yet spawned). They are KEPT
|
||||||
|
/// (not dropped) so the parent edge written at `InitConfig` approval
|
||||||
|
/// survives until the first apply-commit, but they are NOT seeded with a
|
||||||
|
/// default parent here and NOT added to roles — that happens when the
|
||||||
|
/// container actually spawns and the name moves into `agent_names`.
|
||||||
|
pub fn reconcile(agent_names: &[String], pending: &[String]) -> std::io::Result<bool> {
|
||||||
|
let (next, changed) = apply_reconcile(&read(), agent_names, pending);
|
||||||
|
if changed {
|
||||||
|
write(&next)?;
|
||||||
|
}
|
||||||
|
// Keep roles in sync with the live agent set (pending agents get
|
||||||
|
// roles when they spawn).
|
||||||
|
reconcile_roles(agent_names)?;
|
||||||
|
Ok(changed)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pure form of [`reconcile`] for unit tests. Adds missing live agents
|
||||||
|
/// at their default position, drops entries for agents that are neither
|
||||||
|
/// live nor pending-init, and reports whether anything changed.
|
||||||
|
#[must_use]
|
||||||
|
pub fn apply_reconcile(
|
||||||
|
current: &BTreeMap<String, Option<String>>,
|
||||||
|
agent_names: &[String],
|
||||||
|
pending: &[String],
|
||||||
|
) -> (BTreeMap<String, Option<String>>, bool) {
|
||||||
|
let mut next = current.clone();
|
||||||
|
let mut changed = false;
|
||||||
|
for name in agent_names {
|
||||||
|
if !next.contains_key(name) {
|
||||||
|
// A new agent with no declared parent defaults to root
|
||||||
|
// (parent = null). Agent-requested sub-agents always carry an
|
||||||
|
// explicit requester-as-parent edge (written at init_config
|
||||||
|
// approval), so they never hit this default — only
|
||||||
|
// user/operator-initiated spawns do, and those are roots. No
|
||||||
|
// agent is structurally privileged here: "root-ness" is just
|
||||||
|
// a null parent.
|
||||||
|
next.insert(name.clone(), None);
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let known: std::collections::HashSet<&String> =
|
||||||
|
agent_names.iter().chain(pending.iter()).collect();
|
||||||
|
next.retain(|name, _| {
|
||||||
|
let keep = known.contains(name);
|
||||||
|
if !keep {
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
keep
|
||||||
|
});
|
||||||
|
(next, changed)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Roles
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Agents with this role have every parentless agent in the topology
|
||||||
|
/// (see `top_level_agents`) added as virtual children for bind-mount
|
||||||
|
/// and config-change purposes. Enables recovery: if a top-level agent
|
||||||
|
/// is down, a role holder can still read its state and update its config.
|
||||||
|
///
|
||||||
|
/// The manager receives this role by default on first `reconcile_roles`
|
||||||
|
/// call; operators can revoke it with `set_role`.
|
||||||
|
pub const ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS: &str = "can_manage_top_level_agents";
|
||||||
|
|
||||||
|
const ROLES_FILE: &str = "roles.json";
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
|
pub fn roles_path() -> std::path::PathBuf {
|
||||||
|
crate::meta::meta_dir().join(ROLES_FILE)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read the roles map from disk. Returns an empty map when absent or
|
||||||
|
/// unparsable — same safe-degradation pattern as `topology::read`.
|
||||||
|
#[must_use]
|
||||||
|
pub fn read_roles() -> BTreeMap<String, Vec<String>> {
|
||||||
|
let Ok(raw) = std::fs::read_to_string(roles_path()) else {
|
||||||
|
return BTreeMap::new();
|
||||||
|
};
|
||||||
|
serde_json::from_str(&raw).unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Persist the roles map. Sorted output keeps git diffs minimal.
|
||||||
|
pub fn write_roles(roles: &BTreeMap<String, Vec<String>>) -> std::io::Result<()> {
|
||||||
|
let path = roles_path();
|
||||||
|
if let Some(parent) = path.parent() {
|
||||||
|
std::fs::create_dir_all(parent)?;
|
||||||
|
}
|
||||||
|
let text = serde_json::to_string_pretty(roles)
|
||||||
|
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
|
||||||
|
std::fs::write(&path, format!("{text}\n"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return true when `name` holds `role`.
|
||||||
|
#[must_use]
|
||||||
|
pub fn has_role(name: &str, role: &str) -> bool {
|
||||||
|
has_role_in(&read_roles(), name, role)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pure form of [`has_role`] for unit tests.
|
||||||
|
#[must_use]
|
||||||
|
pub fn has_role_in(roles: &BTreeMap<String, Vec<String>>, name: &str, role: &str) -> bool {
|
||||||
|
roles
|
||||||
|
.get(name)
|
||||||
|
.is_some_and(|rs| rs.iter().any(|r| r == role))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Grant or revoke a role for `name`. Idempotent — no disk write when the
|
||||||
|
/// state is already correct.
|
||||||
|
///
|
||||||
|
/// Empty role lists are kept in the map (never removed). An absent key means
|
||||||
|
/// "never seen" (seed on next `reconcile_roles`); an empty list means
|
||||||
|
/// "explicitly revoked" (do not re-seed). Callers that want to remove an
|
||||||
|
/// agent from the map entirely should use `reconcile_roles` (agent departure).
|
||||||
|
pub fn set_role(name: &str, role: &str, enabled: bool) -> Result<(), String> {
|
||||||
|
let mut roles = read_roles();
|
||||||
|
let list = roles.entry(name.to_owned()).or_default();
|
||||||
|
let held = list.iter().any(|r| r == role);
|
||||||
|
match (enabled, held) {
|
||||||
|
(true, false) => list.push(role.to_owned()),
|
||||||
|
(false, true) => list.retain(|r| r != role),
|
||||||
|
_ => return Ok(()),
|
||||||
|
}
|
||||||
|
// Intentionally do NOT remove empty entries — an empty list signals an
|
||||||
|
// explicit revoke and prevents reconcile_roles from re-seeding the role.
|
||||||
|
write_roles(&roles).map_err(|e| format!("write roles.json: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reconcile `roles.json` against the current agent set:
|
||||||
|
/// - Seeds root's default `can_manage_top_level_agents` role on first
|
||||||
|
/// appearance (operator can revoke with `set_role`).
|
||||||
|
/// - Drops entries for agents that no longer exist.
|
||||||
|
///
|
||||||
|
/// Returns true when the file changed.
|
||||||
|
pub fn reconcile_roles(agent_names: &[String]) -> std::io::Result<bool> {
|
||||||
|
let mut roles = read_roles();
|
||||||
|
let mut changed = false;
|
||||||
|
|
||||||
|
let root = crate::lifecycle::MANAGER_NAME;
|
||||||
|
if agent_names.iter().any(|n| n == root) && !roles.contains_key(root) {
|
||||||
|
roles.insert(
|
||||||
|
root.to_owned(),
|
||||||
|
vec![ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS.to_owned()],
|
||||||
|
);
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
let known: std::collections::HashSet<_> = agent_names.iter().collect();
|
||||||
|
roles.retain(|name, _| {
|
||||||
|
let keep = known.contains(name);
|
||||||
|
if !keep {
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
keep
|
||||||
|
});
|
||||||
|
|
||||||
|
if changed {
|
||||||
|
write_roles(&roles)?;
|
||||||
|
}
|
||||||
|
Ok(changed)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn default_seed_makes_every_agent_root() {
|
||||||
|
// No structural manager: every agent defaults to root (null
|
||||||
|
// parent). Explicit edges (init_config / dashboard) are layered
|
||||||
|
// on later.
|
||||||
|
let agents = vec![
|
||||||
|
"alice".to_owned(),
|
||||||
|
crate::lifecycle::MANAGER_NAME.to_owned(),
|
||||||
|
"bob".to_owned(),
|
||||||
|
];
|
||||||
|
let seed = default_seed(&agents);
|
||||||
|
assert_eq!(seed.get(crate::lifecycle::MANAGER_NAME), Some(&None));
|
||||||
|
assert_eq!(seed.get("alice"), Some(&None));
|
||||||
|
assert_eq!(seed.get("bob"), Some(&None));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn default_seed_handles_empty_input() {
|
||||||
|
let seed = default_seed(&[]);
|
||||||
|
assert!(seed.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
fn topo_three_level() -> BTreeMap<String, Option<String>> {
|
||||||
|
let mut m = BTreeMap::new();
|
||||||
|
m.insert(crate::lifecycle::MANAGER_NAME.to_owned(), None);
|
||||||
|
m.insert(
|
||||||
|
"alice".to_owned(),
|
||||||
|
Some(crate::lifecycle::MANAGER_NAME.to_owned()),
|
||||||
|
);
|
||||||
|
m.insert("bob".to_owned(), Some("alice".to_owned()));
|
||||||
|
m.insert("carol".to_owned(), Some("alice".to_owned()));
|
||||||
|
m
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn apply_set_parent_promotes_to_root() {
|
||||||
|
let next = apply_set_parent(&topo_three_level(), "alice", None).unwrap();
|
||||||
|
assert_eq!(next.get("alice"), Some(&None));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn apply_set_parent_reparents_under_sibling_subtree() {
|
||||||
|
// bob and carol both under alice; move carol under bob.
|
||||||
|
let next = apply_set_parent(&topo_three_level(), "carol", Some("bob")).unwrap();
|
||||||
|
assert_eq!(next.get("carol"), Some(&Some("bob".to_owned())));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn apply_set_parent_allows_manager_move() {
|
||||||
|
// The manager is reparentable like any other agent (its
|
||||||
|
// privileges live on the MCP socket, not its tree position).
|
||||||
|
// Build a topo with an unrelated root-level agent `peer` so
|
||||||
|
// moving the manager under it doesn't trip the cycle walk
|
||||||
|
// (every non-manager agent in topo_three_level descends from
|
||||||
|
// the manager, so that fixture can't exercise a legal
|
||||||
|
// manager move).
|
||||||
|
let mut topo = BTreeMap::new();
|
||||||
|
topo.insert(crate::lifecycle::MANAGER_NAME.to_owned(), None);
|
||||||
|
topo.insert("peer".to_owned(), None);
|
||||||
|
let next = apply_set_parent(&topo, crate::lifecycle::MANAGER_NAME, Some("peer"))
|
||||||
|
.expect("manager move should succeed");
|
||||||
|
assert_eq!(
|
||||||
|
next.get(crate::lifecycle::MANAGER_NAME),
|
||||||
|
Some(&Some("peer".to_owned()))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn apply_set_parent_refuses_manager_under_own_descendant() {
|
||||||
|
// Moving the manager under `bob` (who already lives under
|
||||||
|
// `alice` who lives under the manager) would close the loop.
|
||||||
|
// The general cycle walk catches this; no separate manager
|
||||||
|
// guard needed.
|
||||||
|
let err = apply_set_parent(
|
||||||
|
&topo_three_level(),
|
||||||
|
crate::lifecycle::MANAGER_NAME,
|
||||||
|
Some("bob"),
|
||||||
|
)
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(err.contains("cycle"), "err = {err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn apply_set_parent_refuses_unknown_child() {
|
||||||
|
let err = apply_set_parent(&topo_three_level(), "nobody", Some("alice")).unwrap_err();
|
||||||
|
assert!(err.contains("unknown agent"), "err = {err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn apply_set_parent_refuses_unknown_parent() {
|
||||||
|
let err = apply_set_parent(&topo_three_level(), "bob", Some("nobody")).unwrap_err();
|
||||||
|
assert!(err.contains("unknown parent"), "err = {err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn apply_set_parent_refuses_self() {
|
||||||
|
let err = apply_set_parent(&topo_three_level(), "alice", Some("alice")).unwrap_err();
|
||||||
|
assert!(err.contains("own parent"), "err = {err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn apply_set_parent_refuses_cycle() {
|
||||||
|
// bob's parent is alice; trying to make alice's parent =
|
||||||
|
// bob would close the loop alice → bob → alice.
|
||||||
|
let err = apply_set_parent(&topo_three_level(), "alice", Some("bob")).unwrap_err();
|
||||||
|
assert!(err.contains("cycle"), "err = {err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn apply_set_parent_refuses_deep_cycle() {
|
||||||
|
// Three-deep chain: manager → alice → bob → carol. Moving
|
||||||
|
// alice under carol would create the loop alice → carol → bob → alice.
|
||||||
|
let mut topo = BTreeMap::new();
|
||||||
|
topo.insert(crate::lifecycle::MANAGER_NAME.to_owned(), None);
|
||||||
|
topo.insert(
|
||||||
|
"alice".to_owned(),
|
||||||
|
Some(crate::lifecycle::MANAGER_NAME.to_owned()),
|
||||||
|
);
|
||||||
|
topo.insert("bob".to_owned(), Some("alice".to_owned()));
|
||||||
|
topo.insert("carol".to_owned(), Some("bob".to_owned()));
|
||||||
|
let err = apply_set_parent(&topo, "alice", Some("carol")).unwrap_err();
|
||||||
|
assert!(err.contains("cycle"), "err = {err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn apply_set_parent_is_idempotent_noop() {
|
||||||
|
// bob is already under alice — same value returned.
|
||||||
|
let next = apply_set_parent(&topo_three_level(), "bob", Some("alice")).unwrap();
|
||||||
|
assert_eq!(next, topo_three_level());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn apply_add_child_inserts_new_edge_under_parent() {
|
||||||
|
// alice spawns a brand-new child `dora`: the edge lands
|
||||||
|
// with alice as parent without disturbing the rest of the tree.
|
||||||
|
let next = apply_add_child(&topo_three_level(), "dora", "alice")
|
||||||
|
.unwrap()
|
||||||
|
.expect("brand-new edge should produce a map");
|
||||||
|
assert_eq!(next.get("dora"), Some(&Some("alice".to_owned())));
|
||||||
|
// existing entries untouched.
|
||||||
|
assert_eq!(next.get("bob"), Some(&Some("alice".to_owned())));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn apply_add_child_is_idempotent_when_edge_exists() {
|
||||||
|
// bob already under alice — re-init returns a no-op (None).
|
||||||
|
assert!(
|
||||||
|
apply_add_child(&topo_three_level(), "bob", "alice")
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn apply_add_child_refuses_unknown_parent() {
|
||||||
|
let err = apply_add_child(&topo_three_level(), "dora", "nobody").unwrap_err();
|
||||||
|
assert!(err.contains("unknown parent"), "err = {err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn apply_add_child_refuses_name_owned_by_other_parent() {
|
||||||
|
// bob lives under alice; the manager can't claim it via add_child.
|
||||||
|
let err = apply_add_child(&topo_three_level(), "bob", crate::lifecycle::MANAGER_NAME)
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(err.contains("already exists"), "err = {err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn apply_reconcile_adds_missing_live_agent_as_root() {
|
||||||
|
// A live agent with no prior topology entry defaults to root
|
||||||
|
// (null parent) — no structural manager to hang it under.
|
||||||
|
let live = vec![
|
||||||
|
crate::lifecycle::MANAGER_NAME.to_owned(),
|
||||||
|
"newbie".to_owned(),
|
||||||
|
];
|
||||||
|
let (next, changed) = apply_reconcile(&BTreeMap::new(), &live, &[]);
|
||||||
|
assert!(changed);
|
||||||
|
assert_eq!(next.get("newbie"), Some(&None));
|
||||||
|
assert_eq!(next.get(crate::lifecycle::MANAGER_NAME), Some(&None));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn apply_reconcile_drops_vanished_agent() {
|
||||||
|
let live = vec![
|
||||||
|
crate::lifecycle::MANAGER_NAME.to_owned(),
|
||||||
|
"alice".to_owned(),
|
||||||
|
];
|
||||||
|
// carol + bob are gone from the live set and not pending.
|
||||||
|
let (next, changed) = apply_reconcile(&topo_three_level(), &live, &[]);
|
||||||
|
assert!(changed);
|
||||||
|
assert!(!next.contains_key("bob"));
|
||||||
|
assert!(!next.contains_key("carol"));
|
||||||
|
assert!(next.contains_key("alice"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn apply_reconcile_keeps_pending_init_agent_edge() {
|
||||||
|
// `dora` was init'd under alice (edge present) but has no
|
||||||
|
// container yet, so it's absent from the live set. It must NOT
|
||||||
|
// be dropped, and its alice-parent edge must be preserved (not
|
||||||
|
// re-seeded under the manager).
|
||||||
|
let mut topo = topo_three_level();
|
||||||
|
topo.insert("dora".to_owned(), Some("alice".to_owned()));
|
||||||
|
let live = vec![
|
||||||
|
crate::lifecycle::MANAGER_NAME.to_owned(),
|
||||||
|
"alice".to_owned(),
|
||||||
|
"bob".to_owned(),
|
||||||
|
"carol".to_owned(),
|
||||||
|
];
|
||||||
|
let pending = vec!["dora".to_owned()];
|
||||||
|
let (next, changed) = apply_reconcile(&topo, &live, &pending);
|
||||||
|
assert!(!changed, "no change expected: {next:?}");
|
||||||
|
assert_eq!(next.get("dora"), Some(&Some("alice".to_owned())));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resolve_recipient_passes_through_ordinary_names() {
|
||||||
|
let topo = topo_three_level();
|
||||||
|
// Real labels, broadcast, and the operator literal all
|
||||||
|
// shortcut through unchanged — no resolution magic.
|
||||||
|
assert_eq!(resolve_recipient_in(&topo, "bob", "alice"), "alice");
|
||||||
|
assert_eq!(resolve_recipient_in(&topo, "bob", "*"), "*");
|
||||||
|
assert_eq!(
|
||||||
|
resolve_recipient_in(&topo, "bob", hive_sh4re::OPERATOR_RECIPIENT),
|
||||||
|
hive_sh4re::OPERATOR_RECIPIENT
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resolve_recipient_rewrites_parent_sentinel_to_parent_label() {
|
||||||
|
let topo = topo_three_level();
|
||||||
|
// bob's parent is alice → `<parent>` from bob goes to alice.
|
||||||
|
assert_eq!(
|
||||||
|
resolve_recipient_in(&topo, "bob", hive_sh4re::PARENT_RECIPIENT),
|
||||||
|
"alice"
|
||||||
|
);
|
||||||
|
// alice's parent is the manager — same one-hop rewrite.
|
||||||
|
assert_eq!(
|
||||||
|
resolve_recipient_in(&topo, "alice", hive_sh4re::PARENT_RECIPIENT),
|
||||||
|
crate::lifecycle::MANAGER_NAME
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resolve_recipient_falls_back_to_operator_for_root_agent() {
|
||||||
|
let topo = topo_three_level();
|
||||||
|
// Manager is structurally root (parent = None) → `<parent>`
|
||||||
|
// resolves to the operator (the "no parent → tell mara"
|
||||||
|
// fallback documented in conventions.md).
|
||||||
|
assert_eq!(
|
||||||
|
resolve_recipient_in(
|
||||||
|
&topo,
|
||||||
|
crate::lifecycle::MANAGER_NAME,
|
||||||
|
hive_sh4re::PARENT_RECIPIENT
|
||||||
|
),
|
||||||
|
hive_sh4re::OPERATOR_RECIPIENT
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resolve_recipient_falls_back_to_operator_for_unknown_sender() {
|
||||||
|
// Sender absent from topology entirely — defensive fallback
|
||||||
|
// covers the race window where an agent's spawn has registered
|
||||||
|
// its socket but the meta-flake `sync_agents` hasn't yet added
|
||||||
|
// its row.
|
||||||
|
let topo = topo_three_level();
|
||||||
|
assert_eq!(
|
||||||
|
resolve_recipient_in(&topo, "nobody", hive_sh4re::PARENT_RECIPIENT),
|
||||||
|
hive_sh4re::OPERATOR_RECIPIENT
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn children_of_in_returns_direct_descendants() {
|
||||||
|
let topo = topo_three_level();
|
||||||
|
// alice's children: bob, carol.
|
||||||
|
let mut children = children_of_in(&topo, "alice");
|
||||||
|
children.sort();
|
||||||
|
assert_eq!(children, vec!["bob", "carol"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn children_of_in_manager_returns_root_level_agents() {
|
||||||
|
let topo = topo_three_level();
|
||||||
|
// Only alice's parent is manager; bob+carol are under alice.
|
||||||
|
let children = children_of_in(&topo, crate::lifecycle::MANAGER_NAME);
|
||||||
|
assert_eq!(children, vec!["alice"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn children_of_in_leaf_returns_empty() {
|
||||||
|
let topo = topo_three_level();
|
||||||
|
// bob and carol have no children.
|
||||||
|
assert!(children_of_in(&topo, "bob").is_empty());
|
||||||
|
assert!(children_of_in(&topo, "carol").is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn children_of_in_unknown_sender_returns_empty() {
|
||||||
|
let topo = topo_three_level();
|
||||||
|
assert!(children_of_in(&topo, "nobody").is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn top_level_agents_in_returns_parentless_agents() {
|
||||||
|
let topo = topo_three_level();
|
||||||
|
// Only the manager has no parent (alice/bob/carol all have parents).
|
||||||
|
let top = top_level_agents_in(&topo);
|
||||||
|
assert_eq!(top, vec![crate::lifecycle::MANAGER_NAME]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn top_level_agents_in_multi_root_returns_all_parentless() {
|
||||||
|
let mut topo = topo_three_level();
|
||||||
|
// Simulate a second parentless agent alongside the manager.
|
||||||
|
topo.insert("orphan".to_owned(), None);
|
||||||
|
let mut top = top_level_agents_in(&topo);
|
||||||
|
top.sort();
|
||||||
|
let mut expected = vec![crate::lifecycle::MANAGER_NAME, "orphan"];
|
||||||
|
expected.sort_unstable();
|
||||||
|
assert_eq!(top, expected);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn top_level_agents_in_empty_topo_returns_empty() {
|
||||||
|
let topo = BTreeMap::new();
|
||||||
|
assert!(top_level_agents_in(&topo).is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
// Roles tests (no disk I/O — use the pure `has_role_in` / in-memory maps)
|
||||||
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn has_role_in_returns_true_when_role_held() {
|
||||||
|
let mut roles = BTreeMap::new();
|
||||||
|
roles.insert(
|
||||||
|
"alice".to_owned(),
|
||||||
|
vec![ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS.to_owned()],
|
||||||
|
);
|
||||||
|
assert!(has_role_in(
|
||||||
|
&roles,
|
||||||
|
"alice",
|
||||||
|
ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn has_role_in_returns_false_for_absent_agent() {
|
||||||
|
let roles: BTreeMap<String, Vec<String>> = BTreeMap::new();
|
||||||
|
assert!(!has_role_in(
|
||||||
|
&roles,
|
||||||
|
"alice",
|
||||||
|
ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn has_role_in_returns_false_for_empty_list() {
|
||||||
|
let mut roles = BTreeMap::new();
|
||||||
|
roles.insert("alice".to_owned(), vec![]);
|
||||||
|
assert!(!has_role_in(
|
||||||
|
&roles,
|
||||||
|
"alice",
|
||||||
|
ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Revoking a role must leave the key present with an empty list so
|
||||||
|
/// `reconcile_roles` does not re-seed it.
|
||||||
|
#[test]
|
||||||
|
fn set_role_revoke_keeps_empty_entry_as_tombstone() {
|
||||||
|
let mgr = crate::lifecycle::MANAGER_NAME;
|
||||||
|
// Build an in-memory roles map as set_role would see it after granting.
|
||||||
|
let mut roles: BTreeMap<String, Vec<String>> = BTreeMap::new();
|
||||||
|
roles.insert(
|
||||||
|
mgr.to_owned(),
|
||||||
|
vec![ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS.to_owned()],
|
||||||
|
);
|
||||||
|
|
||||||
|
// Simulate the revoke path of set_role (in-memory, no disk).
|
||||||
|
let list = roles.entry(mgr.to_owned()).or_default();
|
||||||
|
let held = list.iter().any(|r| r == ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS);
|
||||||
|
assert!(held);
|
||||||
|
list.retain(|r| r != ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS);
|
||||||
|
|
||||||
|
// Key must still be present (tombstone), just with an empty list.
|
||||||
|
assert!(roles.contains_key(mgr), "empty entry must not be removed");
|
||||||
|
assert!(roles[mgr].is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `reconcile_roles` must not re-seed the manager when its entry exists
|
||||||
|
/// but is empty (operator explicitly revoked the role).
|
||||||
|
#[test]
|
||||||
|
fn reconcile_roles_in_does_not_reseed_after_explicit_revoke() {
|
||||||
|
let mgr = crate::lifecycle::MANAGER_NAME;
|
||||||
|
let agent_names = [mgr.to_owned(), "alice".to_owned()];
|
||||||
|
let mut roles: BTreeMap<String, Vec<String>> = BTreeMap::new();
|
||||||
|
// Tombstone: manager was seen before but all roles were revoked.
|
||||||
|
roles.insert(mgr.to_owned(), vec![]);
|
||||||
|
|
||||||
|
let mgr_present = agent_names.iter().any(|n| n == mgr);
|
||||||
|
let should_seed = mgr_present && !roles.contains_key(mgr);
|
||||||
|
// should_seed must be false because manager key is present (tombstone).
|
||||||
|
assert!(
|
||||||
|
!should_seed,
|
||||||
|
"reconcile_roles must not re-seed an explicit revoke"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `reconcile_roles` seeds the manager on first appearance (no prior entry).
|
||||||
|
#[test]
|
||||||
|
fn reconcile_roles_in_seeds_root_when_absent() {
|
||||||
|
let mgr = crate::lifecycle::MANAGER_NAME;
|
||||||
|
let agent_names = [mgr.to_owned(), "alice".to_owned()];
|
||||||
|
let roles: BTreeMap<String, Vec<String>> = BTreeMap::new(); // empty
|
||||||
|
|
||||||
|
let should_seed = agent_names.iter().any(|n| n == mgr) && !roles.contains_key(mgr);
|
||||||
|
assert!(should_seed, "reconcile_roles must seed manager when absent");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -11,45 +11,45 @@
|
||||||
//! Every module is re-exported `pub` so anything in the crate is
|
//! Every module is re-exported `pub` so anything in the crate is
|
||||||
//! addressable from either binary; the lib doesn't have a curated
|
//! addressable from either binary; the lib doesn't have a curated
|
||||||
//! surface beyond "this is where the modules live".
|
//! surface beyond "this is where the modules live".
|
||||||
|
//!
|
||||||
|
//! Cohesive clusters live in directory submodules (`stores`, `stats`,
|
||||||
|
//! `agent_config`, `workers`); each of their children is re-exported
|
||||||
|
//! at the crate root so pre-existing `crate::broker::…` /
|
||||||
|
//! `hive_c0re::broker::…` paths keep compiling unchanged.
|
||||||
|
|
||||||
pub mod actions;
|
pub mod actions;
|
||||||
pub mod agent_sockets;
|
pub mod agent_config;
|
||||||
pub mod approvals;
|
|
||||||
pub mod audit_log;
|
|
||||||
pub mod auto_update;
|
|
||||||
pub mod broker;
|
|
||||||
pub mod build_logs;
|
|
||||||
pub mod capabilities;
|
|
||||||
pub mod client;
|
pub mod client;
|
||||||
pub mod container_stats;
|
|
||||||
pub mod container_view;
|
pub mod container_view;
|
||||||
pub mod coordinator;
|
pub mod coordinator;
|
||||||
pub mod crash_watch;
|
|
||||||
pub mod dashboard;
|
pub mod dashboard;
|
||||||
pub mod dashboard_events;
|
pub mod dashboard_events;
|
||||||
pub mod db;
|
|
||||||
pub mod flake_check;
|
pub mod flake_check;
|
||||||
pub mod forge;
|
pub mod forge;
|
||||||
pub mod gateway_nginx;
|
pub mod gateway_nginx;
|
||||||
pub mod hive_stats;
|
|
||||||
pub mod host_stats;
|
|
||||||
pub mod job_queue;
|
pub mod job_queue;
|
||||||
pub mod knowledge;
|
|
||||||
pub mod lifecycle;
|
pub mod lifecycle;
|
||||||
pub mod limits;
|
|
||||||
pub mod loose_ends;
|
pub mod loose_ends;
|
||||||
pub mod matrix;
|
pub mod matrix;
|
||||||
pub mod meta;
|
pub mod meta;
|
||||||
pub mod migrate;
|
pub mod migrate;
|
||||||
pub mod operator_questions;
|
|
||||||
pub mod paths;
|
pub mod paths;
|
||||||
pub mod power;
|
|
||||||
pub mod priv_client;
|
pub mod priv_client;
|
||||||
pub mod questions;
|
pub mod questions;
|
||||||
pub mod reminder_scheduler;
|
|
||||||
pub mod scheduled_prompts;
|
|
||||||
pub mod scheduled_prompts_worker;
|
|
||||||
pub mod server;
|
pub mod server;
|
||||||
pub mod socket_server;
|
pub mod socket_server;
|
||||||
pub mod tool_groups;
|
pub mod stats;
|
||||||
pub mod topology;
|
pub mod stores;
|
||||||
|
pub mod workers;
|
||||||
|
|
||||||
|
// Root re-exports: keep every pre-grouping `crate::<module>` /
|
||||||
|
// `hive_c0re::<module>` path compiling without touching consumers.
|
||||||
|
pub use agent_config::{capabilities, limits, tool_groups, topology};
|
||||||
|
pub use stats::{container_stats, hive_stats, host_stats};
|
||||||
|
pub use stores::{
|
||||||
|
approvals, audit_log, broker, build_logs, db, operator_questions, power, scheduled_prompts,
|
||||||
|
};
|
||||||
|
pub use workers::{
|
||||||
|
agent_sockets, auto_update, crash_watch, knowledge, reminder_scheduler,
|
||||||
|
scheduled_prompts_worker,
|
||||||
|
};
|
||||||
|
|
|
||||||
307
hive-c0re/src/stats/container_stats.rs
Normal file
307
hive-c0re/src/stats/container_stats.rs
Normal file
|
|
@ -0,0 +1,307 @@
|
||||||
|
//! Live per-agent-container resource load (CPU + memory) for the
|
||||||
|
//! dashboard. Reads cgroup v2 stat files for each running agent
|
||||||
|
//! machine directly — same privsep-clean posture as the turn-stats
|
||||||
|
//! sqlite reads (`cpu.stat` / `memory.*` are world-readable; no
|
||||||
|
//! `hive-priv` needed). Runs host-side in hive-c0re, where
|
||||||
|
//! `/sys/fs/cgroup/machine.slice/` holds the per-container cgroups. Because
|
||||||
|
//! nixos-container runs `systemd-nspawn --keep-unit` (with
|
||||||
|
//! `Slice = "machine.slice"`), each container's cgroup is its launching
|
||||||
|
//! service unit `container@<machine>.service` — not a machined
|
||||||
|
//! `machine-<name>.scope`. See [`scope_dir`].
|
||||||
|
//!
|
||||||
|
//! No network: agents share the host network namespace
|
||||||
|
//! (`privateNetwork = false`), so there is no per-container net
|
||||||
|
//! counter to read. Per-agent network only becomes meaningful with the
|
||||||
|
//! netns-isolation roadmap (`docs/network.md`).
|
||||||
|
//!
|
||||||
|
//! CPU is cumulative (`usage_usec` is monotonic), so a single read is
|
||||||
|
//! meaningless — we sample every machine's counter, sleep one short
|
||||||
|
//! interval, sample again, and divide the delta by `interval × nproc`
|
||||||
|
//! to get a host-normalised percentage (0..100 across all cores).
|
||||||
|
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::sync::{OnceLock, RwLock};
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use serde::Serialize;
|
||||||
|
use tokio::time::sleep;
|
||||||
|
|
||||||
|
use crate::coordinator::Coordinator;
|
||||||
|
|
||||||
|
/// Sampling interval for the two CPU reads. Short enough to keep the
|
||||||
|
/// endpoint snappy, long enough that the delta isn't dominated by read
|
||||||
|
/// jitter.
|
||||||
|
const CPU_SAMPLE: Duration = Duration::from_millis(200);
|
||||||
|
|
||||||
|
const MACHINE_SLICE: &str = "/sys/fs/cgroup/machine.slice";
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct ContainerResource {
|
||||||
|
/// Agent name (without the `h-` machine prefix).
|
||||||
|
pub name: String,
|
||||||
|
/// Host-normalised CPU usage over the sample interval, as a
|
||||||
|
/// percentage of total host CPU (0..100 across all cores).
|
||||||
|
pub cpu_pct: f64,
|
||||||
|
/// Current memory usage (`memory.current`), bytes.
|
||||||
|
pub mem_current_bytes: u64,
|
||||||
|
/// High-water memory usage since container start (`memory.peak`),
|
||||||
|
/// bytes. `None` if the kernel doesn't expose `memory.peak`.
|
||||||
|
pub mem_peak_bytes: Option<u64>,
|
||||||
|
/// Memory ceiling (`memory.max`), bytes. `None` when unlimited
|
||||||
|
/// (the file reads `max`).
|
||||||
|
pub mem_max_bytes: Option<u64>,
|
||||||
|
/// Last-sampled total disk usage — the agent's state dir plus the
|
||||||
|
/// container's writable rootfs, excluding the shared read-only nix
|
||||||
|
/// store — in bytes. `None` until the slow background sampler
|
||||||
|
/// ([`disk_sampler_loop`]) has populated the cache at least once.
|
||||||
|
/// Decoupled from this hot per-poll path because a `du` tree-walk is
|
||||||
|
/// far too expensive at the 5s `/api/container-load` cadence.
|
||||||
|
pub disk_bytes: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Cadence for the slow disk sampler. Disk size changes slowly relative
|
||||||
|
/// to CPU/mem, and a `du` walk is expensive, so this runs far less often
|
||||||
|
/// than the per-poll cgroup reads — the row just carries the last value.
|
||||||
|
const DISK_SAMPLE_INTERVAL: Duration = Duration::from_mins(5);
|
||||||
|
|
||||||
|
/// Root of the per-container writable rootfs trees that nixos-containers
|
||||||
|
/// manages (`/var/lib/nixos-containers/<machine>`).
|
||||||
|
const NIXOS_CONTAINERS_ROOT: &str = "/var/lib/nixos-containers";
|
||||||
|
|
||||||
|
/// Last-sampled per-agent disk usage (bytes), keyed by agent name.
|
||||||
|
/// Written by [`disk_sampler_loop`] (~every 5 min), read by [`gather`]
|
||||||
|
/// so the hot poll never runs `du`.
|
||||||
|
fn disk_cache() -> &'static RwLock<HashMap<String, u64>> {
|
||||||
|
static CACHE: OnceLock<RwLock<HashMap<String, u64>>> = OnceLock::new();
|
||||||
|
CACHE.get_or_init(|| RwLock::new(HashMap::new()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `du -sxb <path>` → apparent total bytes. `-s` summary, `-b` apparent
|
||||||
|
/// size, `-x` stay on one filesystem so the walk never descends into the
|
||||||
|
/// bind-mounted read-only shared nix store (or any other bind mount) —
|
||||||
|
/// exactly the per-container-only measure we want. Best-effort: a
|
||||||
|
/// missing/unreadable path (e.g. a stopped container with no rootfs)
|
||||||
|
/// yields `None`, which the caller treats as 0.
|
||||||
|
async fn du_bytes(path: &std::path::Path) -> Option<u64> {
|
||||||
|
let out = tokio::process::Command::new("du")
|
||||||
|
.arg("-sxb")
|
||||||
|
.arg(path)
|
||||||
|
.output()
|
||||||
|
.await
|
||||||
|
.ok()?;
|
||||||
|
if !out.status.success() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let stdout = String::from_utf8_lossy(&out.stdout);
|
||||||
|
stdout.split_whitespace().next()?.parse::<u64>().ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Total per-agent disk: the agent's host-side state dir
|
||||||
|
/// (`/var/lib/hyperhive/agents/<name>/state`) plus the container's writable
|
||||||
|
/// rootfs (`/var/lib/nixos-containers/h-<name>`). Each measured with `du -sxb`
|
||||||
|
/// so the shared nix store and other bind mounts are excluded. A path that
|
||||||
|
/// doesn't exist contributes 0.
|
||||||
|
///
|
||||||
|
/// The state dir is resolved via [`Coordinator::agent_notes_dir`] — the same
|
||||||
|
/// host path the dashboard reads agent state from. Hardcoding the
|
||||||
|
/// container-internal bind-mount path (`/agents/<name>/state`) instead made
|
||||||
|
/// `du` fail host-side (that path only exists inside the container), so every
|
||||||
|
/// agent's state-dir contribution was 0; with the writable rootfs nearly empty
|
||||||
|
/// (almost everything is bind-mounted), that surfaced as all agents reporting
|
||||||
|
/// 0 disk.
|
||||||
|
async fn measure_agent_disk(name: &str) -> u64 {
|
||||||
|
let state_dir = Coordinator::agent_notes_dir(name);
|
||||||
|
let rootfs = PathBuf::from(format!("{NIXOS_CONTAINERS_ROOT}/h-{name}"));
|
||||||
|
let mut total = 0u64;
|
||||||
|
for path in [state_dir, rootfs] {
|
||||||
|
if let Some(bytes) = du_bytes(&path).await {
|
||||||
|
total = total.saturating_add(bytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
total
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Background loop: every [`DISK_SAMPLE_INTERVAL`], recompute disk usage
|
||||||
|
/// for every kept-state agent and refresh [`disk_cache`]. The first pass
|
||||||
|
/// runs immediately so the dashboard shows disk shortly after boot.
|
||||||
|
/// Runs forever; spawn once at hive-c0re startup via [`spawn_disk_sampler`].
|
||||||
|
pub async fn disk_sampler_loop() {
|
||||||
|
loop {
|
||||||
|
for name in Coordinator::kept_state_names() {
|
||||||
|
let bytes = measure_agent_disk(&name).await;
|
||||||
|
if let Ok(mut cache) = disk_cache().write() {
|
||||||
|
cache.insert(name, bytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sleep(DISK_SAMPLE_INTERVAL).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Spawn the slow disk sampler as a detached background task. Called once
|
||||||
|
/// from `main` alongside the other host-side loops.
|
||||||
|
pub fn spawn_disk_sampler() {
|
||||||
|
tokio::spawn(disk_sampler_loop());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Cgroup directory for a container's machine.
|
||||||
|
///
|
||||||
|
/// nixos-container runs `systemd-nspawn --keep-unit` with
|
||||||
|
/// `Slice = "machine.slice"` (see nixpkgs
|
||||||
|
/// `virtualisation/nixos-containers.nix`). `--keep-unit` means nspawn does
|
||||||
|
/// **not** create a separate machined `machine-<name>.scope` — the
|
||||||
|
/// container's cgroup *is* the launching service unit,
|
||||||
|
/// `container@<machine>.service`, placed under `machine.slice`.
|
||||||
|
/// systemd-machined still logs "New machine <name>" (registration), but the
|
||||||
|
/// cgroup stays on the service unit. So the path is
|
||||||
|
/// `machine.slice/container@<machine>.service`, and the service unit name is
|
||||||
|
/// used verbatim — no `\x2d` escaping (that only applies when a string is
|
||||||
|
/// converted *into* a scope/slice unit name, not to an already-formed
|
||||||
|
/// instance unit; the journal shows the literal `container@h-<agent>.service`).
|
||||||
|
fn scope_dir(machine: &str) -> PathBuf {
|
||||||
|
PathBuf::from(MACHINE_SLICE).join(format!("container@{machine}.service"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read a single unsigned integer from a one-line cgroup file.
|
||||||
|
fn read_u64(path: &std::path::Path) -> Option<u64> {
|
||||||
|
std::fs::read_to_string(path)
|
||||||
|
.ok()?
|
||||||
|
.trim()
|
||||||
|
.parse::<u64>()
|
||||||
|
.ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `memory.max` reads `max` when there's no limit — map that to `None`.
|
||||||
|
fn read_mem_max(path: &std::path::Path) -> Option<u64> {
|
||||||
|
let s = std::fs::read_to_string(path).ok()?;
|
||||||
|
let s = s.trim();
|
||||||
|
if s == "max" {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
s.parse::<u64>().ok()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse `usage_usec` (cumulative CPU time, microseconds) from a
|
||||||
|
/// scope's `cpu.stat`.
|
||||||
|
fn read_usage_usec(dir: &std::path::Path) -> Option<u64> {
|
||||||
|
let text = std::fs::read_to_string(dir.join("cpu.stat")).ok()?;
|
||||||
|
for line in text.lines() {
|
||||||
|
if let Some(rest) = line.strip_prefix("usage_usec ") {
|
||||||
|
return rest.trim().parse::<u64>().ok();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Total online host CPUs, for normalising CPU% to "% of total host".
|
||||||
|
/// Reads `/sys/devices/system/cpu/online` (a comma-separated list of
|
||||||
|
/// ranges, e.g. `0-19`) rather than `available_parallelism()`, which
|
||||||
|
/// reflects the hive-core process's CPU affinity — if hive-core is ever
|
||||||
|
/// affinity-pinned to a subset, that would under-count the denominator
|
||||||
|
/// and inflate the percentage. Falls back to the process count, then 1.
|
||||||
|
fn host_nproc() -> usize {
|
||||||
|
fn from_sysfs() -> Option<usize> {
|
||||||
|
let s = std::fs::read_to_string("/sys/devices/system/cpu/online").ok()?;
|
||||||
|
let mut count = 0usize;
|
||||||
|
for part in s.trim().split(',') {
|
||||||
|
let part = part.trim();
|
||||||
|
if part.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if let Some((a, b)) = part.split_once('-') {
|
||||||
|
let a: usize = a.trim().parse().ok()?;
|
||||||
|
let b: usize = b.trim().parse().ok()?;
|
||||||
|
count += b.saturating_sub(a) + 1;
|
||||||
|
} else {
|
||||||
|
part.parse::<usize>().ok()?;
|
||||||
|
count += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(count > 0).then_some(count)
|
||||||
|
}
|
||||||
|
from_sysfs()
|
||||||
|
.or_else(|| std::thread::available_parallelism().ok().map(usize::from))
|
||||||
|
.unwrap_or(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Sample CPU + memory for every running agent container. Best-effort:
|
||||||
|
/// agents whose scope dir is absent (not running) or unreadable are
|
||||||
|
/// skipped, never fatal. Sorted by name for a stable display order.
|
||||||
|
pub async fn gather() -> Vec<ContainerResource> {
|
||||||
|
// Machines that currently have a cgroup scope (= running). Agent
|
||||||
|
// machine name is `h-<state-dir name>`.
|
||||||
|
let candidates: Vec<(String, PathBuf)> = Coordinator::kept_state_names()
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|name| {
|
||||||
|
let dir = scope_dir(&format!("h-{name}"));
|
||||||
|
dir.join("cpu.stat").exists().then_some((name, dir))
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
// First CPU sample for all, then one shared sleep, then the second
|
||||||
|
// — so N agents cost one interval, not N.
|
||||||
|
let t0: Vec<Option<u64>> = candidates.iter().map(|(_, d)| read_usage_usec(d)).collect();
|
||||||
|
sleep(CPU_SAMPLE).await;
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
clippy::cast_precision_loss,
|
||||||
|
reason = "CPU-utilisation math; the operands (core count, microsecond sample interval, cgroup time delta) stay well under f64's 2^53 exact-integer range, so no precision is lost"
|
||||||
|
)]
|
||||||
|
let nproc = host_nproc() as f64;
|
||||||
|
#[allow(
|
||||||
|
clippy::cast_precision_loss,
|
||||||
|
reason = "CPU-utilisation math; the operands (core count, microsecond sample interval, cgroup time delta) stay well under f64's 2^53 exact-integer range, so no precision is lost"
|
||||||
|
)]
|
||||||
|
let interval_usec = CPU_SAMPLE.as_micros() as f64;
|
||||||
|
|
||||||
|
// Snapshot the slow disk cache once (cheap clone of a small map) so
|
||||||
|
// the per-agent loop below is a plain map lookup — no `du` on this
|
||||||
|
// hot path. Agents not yet sampled simply carry `disk_bytes = None`.
|
||||||
|
let disk = disk_cache().read().map(|c| c.clone()).unwrap_or_default();
|
||||||
|
|
||||||
|
let mut out: Vec<ContainerResource> = Vec::with_capacity(candidates.len());
|
||||||
|
for (i, (name, dir)) in candidates.iter().enumerate() {
|
||||||
|
let cpu_pct = match (t0[i], read_usage_usec(dir)) {
|
||||||
|
(Some(a), Some(b)) => {
|
||||||
|
#[allow(
|
||||||
|
clippy::cast_precision_loss,
|
||||||
|
reason = "CPU-utilisation math; the operands (core count, microsecond sample interval, cgroup time delta) stay well under f64's 2^53 exact-integer range, so no precision is lost"
|
||||||
|
)]
|
||||||
|
let delta = b.saturating_sub(a) as f64;
|
||||||
|
(delta / (interval_usec * nproc)) * 100.0
|
||||||
|
}
|
||||||
|
_ => 0.0,
|
||||||
|
};
|
||||||
|
out.push(ContainerResource {
|
||||||
|
name: name.clone(),
|
||||||
|
cpu_pct,
|
||||||
|
mem_current_bytes: read_u64(&dir.join("memory.current")).unwrap_or(0),
|
||||||
|
mem_peak_bytes: read_u64(&dir.join("memory.peak")),
|
||||||
|
mem_max_bytes: read_mem_max(&dir.join("memory.max")),
|
||||||
|
disk_bytes: disk.get(name).copied(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn scope_dir_is_the_keep_unit_service_under_machine_slice() {
|
||||||
|
// nixos-container `--keep-unit` keeps the cgroup on the service
|
||||||
|
// unit `container@<machine>.service` (literal name, no `\x2d`),
|
||||||
|
// under machine.slice — NOT a machined `machine-<name>.scope`.
|
||||||
|
assert_eq!(
|
||||||
|
scope_dir("h-atlas"),
|
||||||
|
PathBuf::from("/sys/fs/cgroup/machine.slice/container@h-atlas.service")
|
||||||
|
);
|
||||||
|
// Underscores in agent names are likewise verbatim.
|
||||||
|
assert_eq!(
|
||||||
|
scope_dir("h-foo_bar"),
|
||||||
|
PathBuf::from("/sys/fs/cgroup/machine.slice/container@h-foo_bar.service")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
450
hive-c0re/src/stats/hive_stats.rs
Normal file
450
hive-c0re/src/stats/hive_stats.rs
Normal file
|
|
@ -0,0 +1,450 @@
|
||||||
|
//! Hive-wide turn-stats aggregation for the dashboard's swarm stats
|
||||||
|
//! view. Reads every agent's per-agent
|
||||||
|
//! `hyperhive-turn-stats.sqlite` read-only and rolls the rows up into
|
||||||
|
//! swarm totals + a per-agent rollup + model mix + a *labelled*
|
||||||
|
//! cost estimate.
|
||||||
|
//!
|
||||||
|
//! Why re-read the rows here instead of reusing `hive-ag3nt`'s
|
||||||
|
//! `stats.rs`: that module lives in a different crate (the agent
|
||||||
|
//! harness) which hive-c0re can't import. The stable contract is the
|
||||||
|
//! turn-stats *schema*, so we run a focused query against it. If we
|
||||||
|
//! ever want a single source of truth, the row-read + aggregation can
|
||||||
|
//! be lifted into a shared crate — overkill for now.
|
||||||
|
//!
|
||||||
|
//! Privsep: the sqlite files are mode 0644 owned by the agent user;
|
||||||
|
//! `hive-core` can read them fine, but cannot write/delete (which is why
|
||||||
|
//! retention sweeps run agent-side in the harness, not here). We open
|
||||||
|
//! read-only so an in-flight harness writer never blocks us.
|
||||||
|
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::path::Path;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use rusqlite::{Connection, OpenFlags};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use crate::coordinator::Coordinator;
|
||||||
|
use hive_sh4re::wire_time::now_unix;
|
||||||
|
|
||||||
|
/// Window accepted by `/api/stats-hive?window=`. Maps to a lookback
|
||||||
|
/// span; the hive view is a flat rollup (no per-bucket trend — the
|
||||||
|
/// per-agent `/stats` page owns the trend charts).
|
||||||
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
pub enum Window {
|
||||||
|
Hour,
|
||||||
|
FourHour,
|
||||||
|
Day,
|
||||||
|
ThreeDay,
|
||||||
|
Week,
|
||||||
|
Month,
|
||||||
|
/// All available data: aggregate over every recorded turn since the
|
||||||
|
/// earliest (`from == 0`), with no fixed lookback.
|
||||||
|
All,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Window {
|
||||||
|
#[must_use]
|
||||||
|
pub fn parse(s: &str) -> Self {
|
||||||
|
match s {
|
||||||
|
"1h" => Self::Hour,
|
||||||
|
"4h" => Self::FourHour,
|
||||||
|
"3d" => Self::ThreeDay,
|
||||||
|
"7d" => Self::Week,
|
||||||
|
"30d" => Self::Month,
|
||||||
|
"all" => Self::All,
|
||||||
|
_ => Self::Day,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn label(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Hour => "1h",
|
||||||
|
Self::FourHour => "4h",
|
||||||
|
Self::Day => "24h",
|
||||||
|
Self::ThreeDay => "3d",
|
||||||
|
Self::Week => "7d",
|
||||||
|
Self::Month => "30d",
|
||||||
|
Self::All => "all",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn span_secs(self) -> i64 {
|
||||||
|
match self {
|
||||||
|
Self::Hour => 3600,
|
||||||
|
Self::FourHour => 4 * 3600,
|
||||||
|
Self::Day => 24 * 3600,
|
||||||
|
Self::ThreeDay => 3 * 24 * 3600,
|
||||||
|
Self::Week => 7 * 24 * 3600,
|
||||||
|
Self::Month => 30 * 24 * 3600,
|
||||||
|
// `All` has no fixed lookback; `hive_snapshot` uses `from == 0`
|
||||||
|
// (every recorded turn). 0 here is a consistent safe fallback.
|
||||||
|
Self::All => 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Approximate USD price per **million** tokens for one model class.
|
||||||
|
/// The four fields map to the four token kinds the turn-stats schema
|
||||||
|
/// records. This is a deliberately rough estimate — Anthropic list
|
||||||
|
/// pricing drifts, so the dashboard labels the figure as an estimate.
|
||||||
|
///
|
||||||
|
/// Operators keep the table current without a code change via the
|
||||||
|
/// `services.hyperhive.modelPrices` nix option (passed to
|
||||||
|
/// `hive-c0re serve --model-prices <json>`, held on the `Coordinator`
|
||||||
|
/// as a [`PriceTable`]). Any model not covered by the operator table
|
||||||
|
/// falls back to [`builtin_prices`].
|
||||||
|
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
|
||||||
|
pub struct Prices {
|
||||||
|
pub input: f64,
|
||||||
|
pub output: f64,
|
||||||
|
pub cache_read: f64,
|
||||||
|
pub cache_write: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Operator-tunable model→price map. Keys are matched
|
||||||
|
/// case-insensitively as a substring of the model id (e.g. `"sonnet"`
|
||||||
|
/// matches `"claude-sonnet-4-5"`); the longest matching key wins so a
|
||||||
|
/// specific entry beats a generic family name. An empty map means
|
||||||
|
/// every model uses [`builtin_prices`].
|
||||||
|
pub type PriceTable = HashMap<String, Prices>;
|
||||||
|
|
||||||
|
/// Built-in fallback pricing, per **million** tokens. Used when the
|
||||||
|
/// operator's [`PriceTable`] has no key matching the model. Figures are
|
||||||
|
/// current Anthropic list prices for the Claude 4.x family (Opus 4.x,
|
||||||
|
/// Sonnet 4.x, Haiku 4.5); `cache_write` uses the 1-hour cache-TTL price,
|
||||||
|
/// which is the default through the Claude subscription the agents run
|
||||||
|
/// on. Keep in sync with the `services.hyperhive.modelPrices` nix default
|
||||||
|
/// (`nix/modules/hive-c0re.nix`).
|
||||||
|
fn builtin_prices(model: &str) -> Prices {
|
||||||
|
let m = model.to_ascii_lowercase();
|
||||||
|
if m.contains("opus") {
|
||||||
|
Prices {
|
||||||
|
input: 5.0,
|
||||||
|
output: 25.0,
|
||||||
|
cache_read: 0.5,
|
||||||
|
cache_write: 10.0,
|
||||||
|
}
|
||||||
|
} else if m.contains("haiku") {
|
||||||
|
Prices {
|
||||||
|
input: 1.0,
|
||||||
|
output: 5.0,
|
||||||
|
cache_read: 0.1,
|
||||||
|
cache_write: 2.0,
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// sonnet + unknown fallback
|
||||||
|
Prices {
|
||||||
|
input: 3.0,
|
||||||
|
output: 15.0,
|
||||||
|
cache_read: 0.3,
|
||||||
|
cache_write: 6.0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolve the price for `model`: prefer the operator-provided
|
||||||
|
/// `table` (longest matching key wins, so a specific
|
||||||
|
/// `claude-3-5-sonnet` entry beats a generic `sonnet`), else fall back
|
||||||
|
/// to [`builtin_prices`].
|
||||||
|
fn resolve_prices(model: &str, table: &PriceTable) -> Prices {
|
||||||
|
let m = model.to_ascii_lowercase();
|
||||||
|
let mut best: Option<(usize, Prices)> = None;
|
||||||
|
for (k, v) in table {
|
||||||
|
let key = k.to_ascii_lowercase();
|
||||||
|
if key.is_empty() || !m.contains(&key) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let better = match best {
|
||||||
|
Some((blen, _)) => key.len() > blen,
|
||||||
|
None => true,
|
||||||
|
};
|
||||||
|
if better {
|
||||||
|
best = Some((key.len(), *v));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
match best {
|
||||||
|
Some((_, p)) => p,
|
||||||
|
None => builtin_prices(&m),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct KeyCount {
|
||||||
|
pub key: String,
|
||||||
|
pub count: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct AgentRollup {
|
||||||
|
pub name: String,
|
||||||
|
pub turns: u64,
|
||||||
|
pub input_tokens: u64,
|
||||||
|
pub output_tokens: u64,
|
||||||
|
pub cache_read_tokens: u64,
|
||||||
|
pub cache_creation_tokens: u64,
|
||||||
|
/// Labelled estimate — see [`Prices`].
|
||||||
|
pub est_cost_usd: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct HiveStats {
|
||||||
|
pub window: &'static str,
|
||||||
|
pub from: i64,
|
||||||
|
pub now: i64,
|
||||||
|
/// Number of agents that had at least one turn in the window.
|
||||||
|
pub active_agents: u64,
|
||||||
|
pub total_turns: u64,
|
||||||
|
pub total_input_tokens: u64,
|
||||||
|
pub total_output_tokens: u64,
|
||||||
|
pub total_cache_read_tokens: u64,
|
||||||
|
pub total_cache_creation_tokens: u64,
|
||||||
|
/// Labelled estimate — see [`Prices`].
|
||||||
|
pub est_cost_usd: f64,
|
||||||
|
/// Per-agent rollup, busiest (most turns) first.
|
||||||
|
pub agents: Vec<AgentRollup>,
|
||||||
|
/// Turns per model across the whole swarm, busiest first.
|
||||||
|
pub model_mix: Vec<KeyCount>,
|
||||||
|
/// Most-run normalised bash-command heads ("favorite tools") across
|
||||||
|
/// the whole swarm, busiest first, capped to 10. Empty until the
|
||||||
|
/// hive-bash-mcp capture has recorded `bash_commands` rows on at
|
||||||
|
/// least one active agent.
|
||||||
|
pub bash_mix: Vec<KeyCount>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct AgentAgg {
|
||||||
|
turns: u64,
|
||||||
|
input: u64,
|
||||||
|
output: u64,
|
||||||
|
cache_read: u64,
|
||||||
|
cache_creation: u64,
|
||||||
|
cost: f64,
|
||||||
|
models: HashMap<String, u64>,
|
||||||
|
/// Normalised bash-command head → invocation count, from the agent's
|
||||||
|
/// `bash_commands` table (written by hive-bash-mcp). Empty when that
|
||||||
|
/// capture hasn't run for this agent (table absent).
|
||||||
|
bash: HashMap<String, u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(
|
||||||
|
clippy::cast_sign_loss,
|
||||||
|
clippy::cast_possible_truncation,
|
||||||
|
reason = "v.max(0) clamps to a non-negative value first, so the i64->u64 cast is exact: no sign loss, and no truncation since u64 covers all non-negative i64"
|
||||||
|
)]
|
||||||
|
fn u64_from_i64(v: i64) -> u64 {
|
||||||
|
v.max(0) as u64
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Aggregate one agent's turn-stats over `[from, now]`. Errors bubble
|
||||||
|
/// up so the caller can skip a single bad/locked db without failing
|
||||||
|
/// the whole endpoint.
|
||||||
|
fn read_agent(path: &Path, from: i64, prices: &PriceTable) -> rusqlite::Result<AgentAgg> {
|
||||||
|
let conn = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
|
||||||
|
// turn_stats is rollback-journal (not WAL): a read landing while the
|
||||||
|
// harness is mid-INSERT would get SQLITE_BUSY and drop that active
|
||||||
|
// agent from the rollup. Wait out the brief write instead.
|
||||||
|
conn.busy_timeout(Duration::from_millis(500))?;
|
||||||
|
let mut stmt = conn.prepare(
|
||||||
|
"SELECT model, input_tokens, output_tokens,
|
||||||
|
cache_read_input_tokens, cache_creation_input_tokens
|
||||||
|
FROM turn_stats
|
||||||
|
WHERE started_at >= ?1",
|
||||||
|
)?;
|
||||||
|
let mut agg = AgentAgg::default();
|
||||||
|
let rows = stmt.query_map([from], |row| {
|
||||||
|
Ok((
|
||||||
|
row.get::<_, String>(0)?,
|
||||||
|
u64_from_i64(row.get::<_, i64>(1)?),
|
||||||
|
u64_from_i64(row.get::<_, i64>(2)?),
|
||||||
|
u64_from_i64(row.get::<_, i64>(3)?),
|
||||||
|
u64_from_i64(row.get::<_, i64>(4)?),
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
for r in rows {
|
||||||
|
let (model, input, output, cache_read, cache_creation) = r?;
|
||||||
|
agg.turns += 1;
|
||||||
|
agg.input = agg.input.saturating_add(input);
|
||||||
|
agg.output = agg.output.saturating_add(output);
|
||||||
|
agg.cache_read = agg.cache_read.saturating_add(cache_read);
|
||||||
|
agg.cache_creation = agg.cache_creation.saturating_add(cache_creation);
|
||||||
|
let p = resolve_prices(&model, prices);
|
||||||
|
#[allow(
|
||||||
|
clippy::cast_precision_loss,
|
||||||
|
reason = "token counts stay well under f64's 2^53 exact-integer range, so this cost computation loses no precision"
|
||||||
|
)]
|
||||||
|
{
|
||||||
|
agg.cost += (input as f64 * p.input
|
||||||
|
+ output as f64 * p.output
|
||||||
|
+ cache_read as f64 * p.cache_read
|
||||||
|
+ cache_creation as f64 * p.cache_write)
|
||||||
|
/ 1_000_000.0;
|
||||||
|
}
|
||||||
|
*agg.models.entry(model).or_insert(0) += 1;
|
||||||
|
}
|
||||||
|
agg.bash = read_bash_heads(&conn, from);
|
||||||
|
Ok(agg)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Tally normalised bash-command heads from the agent's `bash_commands`
|
||||||
|
/// table (`ts INTEGER, head TEXT`, written by hive-bash-mcp) over
|
||||||
|
/// `[from, now]`. Best-effort + isolated from `read_agent`'s error path:
|
||||||
|
/// a missing table (capture hasn't run for this agent) or any read error
|
||||||
|
/// yields an empty map rather than propagating, so the favorite-tools
|
||||||
|
/// rollup simply omits that agent and never fails the whole endpoint.
|
||||||
|
/// `ts` is unix seconds, matching the `from` cutoff.
|
||||||
|
fn read_bash_heads(conn: &Connection, from: i64) -> HashMap<String, u64> {
|
||||||
|
let mut out: HashMap<String, u64> = HashMap::new();
|
||||||
|
let Ok(mut stmt) =
|
||||||
|
conn.prepare("SELECT head, COUNT(*) FROM bash_commands WHERE ts >= ?1 GROUP BY head")
|
||||||
|
else {
|
||||||
|
return out;
|
||||||
|
};
|
||||||
|
let Ok(rows) = stmt.query_map([from], |row| {
|
||||||
|
Ok((
|
||||||
|
row.get::<_, String>(0)?,
|
||||||
|
u64_from_i64(row.get::<_, i64>(1)?),
|
||||||
|
))
|
||||||
|
}) else {
|
||||||
|
return out;
|
||||||
|
};
|
||||||
|
for (head, count) in rows.flatten() {
|
||||||
|
*out.entry(head).or_insert(0) += count;
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build the swarm-wide rollup. Best-effort: a missing or unreadable
|
||||||
|
/// per-agent db is skipped (logged), never fatal.
|
||||||
|
#[must_use]
|
||||||
|
pub fn hive_snapshot(window: Window, prices: &PriceTable) -> HiveStats {
|
||||||
|
let now = now_unix();
|
||||||
|
// Fixed windows look back a constant span; `all` aggregates every
|
||||||
|
// recorded turn (`from == 0`). The hive rollup isn't time-bucketed, so
|
||||||
|
// unlike the per-agent snapshot it needs no adaptive bucket sizing.
|
||||||
|
let from = match window {
|
||||||
|
Window::All => 0,
|
||||||
|
_ => now - window.span_secs(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut agents: Vec<AgentRollup> = Vec::new();
|
||||||
|
let mut model_mix: HashMap<String, u64> = HashMap::new();
|
||||||
|
let mut bash_mix: HashMap<String, u64> = HashMap::new();
|
||||||
|
let mut total_turns = 0u64;
|
||||||
|
let mut total_input = 0u64;
|
||||||
|
let mut total_output = 0u64;
|
||||||
|
let mut total_cache_read = 0u64;
|
||||||
|
let mut total_cache_creation = 0u64;
|
||||||
|
let mut total_cost = 0.0f64;
|
||||||
|
let mut active_agents = 0u64;
|
||||||
|
|
||||||
|
for name in Coordinator::kept_state_names() {
|
||||||
|
let path = Coordinator::agent_harness_dir(&name).join("hyperhive-turn-stats.sqlite");
|
||||||
|
if !path.exists() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let agg = match read_agent(&path, from, prices) {
|
||||||
|
Ok(a) => a,
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!(agent = %name, error = ?e, "hive-stats: read failed; skipping");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if agg.turns == 0 {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
active_agents += 1;
|
||||||
|
total_turns += agg.turns;
|
||||||
|
total_input = total_input.saturating_add(agg.input);
|
||||||
|
total_output = total_output.saturating_add(agg.output);
|
||||||
|
total_cache_read = total_cache_read.saturating_add(agg.cache_read);
|
||||||
|
total_cache_creation = total_cache_creation.saturating_add(agg.cache_creation);
|
||||||
|
total_cost += agg.cost;
|
||||||
|
for (m, c) in &agg.models {
|
||||||
|
*model_mix.entry(m.clone()).or_insert(0) += c;
|
||||||
|
}
|
||||||
|
for (h, c) in &agg.bash {
|
||||||
|
*bash_mix.entry(h.clone()).or_insert(0) += c;
|
||||||
|
}
|
||||||
|
agents.push(AgentRollup {
|
||||||
|
name,
|
||||||
|
turns: agg.turns,
|
||||||
|
input_tokens: agg.input,
|
||||||
|
output_tokens: agg.output,
|
||||||
|
cache_read_tokens: agg.cache_read,
|
||||||
|
cache_creation_tokens: agg.cache_creation,
|
||||||
|
est_cost_usd: agg.cost,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Busiest agents first.
|
||||||
|
agents.sort_by(|a, b| b.turns.cmp(&a.turns).then_with(|| a.name.cmp(&b.name)));
|
||||||
|
|
||||||
|
let mut model_mix: Vec<KeyCount> = model_mix
|
||||||
|
.into_iter()
|
||||||
|
.map(|(key, count)| KeyCount { key, count })
|
||||||
|
.collect();
|
||||||
|
model_mix.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.key.cmp(&b.key)));
|
||||||
|
|
||||||
|
// Busiest commands first, capped to a top-10 "favorite tools" list.
|
||||||
|
let mut bash_mix: Vec<KeyCount> = bash_mix
|
||||||
|
.into_iter()
|
||||||
|
.map(|(key, count)| KeyCount { key, count })
|
||||||
|
.collect();
|
||||||
|
bash_mix.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.key.cmp(&b.key)));
|
||||||
|
bash_mix.truncate(10);
|
||||||
|
|
||||||
|
HiveStats {
|
||||||
|
window: window.label(),
|
||||||
|
from,
|
||||||
|
now,
|
||||||
|
active_agents,
|
||||||
|
total_turns,
|
||||||
|
total_input_tokens: total_input,
|
||||||
|
total_output_tokens: total_output,
|
||||||
|
total_cache_read_tokens: total_cache_read,
|
||||||
|
total_cache_creation_tokens: total_cache_creation,
|
||||||
|
est_cost_usd: total_cost,
|
||||||
|
agents,
|
||||||
|
model_mix,
|
||||||
|
bash_mix,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// `read_bash_heads` tallies per-head counts over the window and is
|
||||||
|
/// isolated from a missing table (returns empty, never errors).
|
||||||
|
#[test]
|
||||||
|
fn bash_heads_tally_and_window() {
|
||||||
|
let conn = Connection::open_in_memory().unwrap();
|
||||||
|
conn.execute_batch("CREATE TABLE bash_commands (ts INTEGER NOT NULL, head TEXT NOT NULL);")
|
||||||
|
.unwrap();
|
||||||
|
let now = now_unix();
|
||||||
|
for (ts, head) in [
|
||||||
|
(now - 100, "cargo"),
|
||||||
|
(now - 200, "cargo"),
|
||||||
|
(now - 300, "git"),
|
||||||
|
(now - 10_000, "rg"), // outside a 1h-ish cutoff below
|
||||||
|
] {
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO bash_commands (ts, head) VALUES (?1, ?2)",
|
||||||
|
rusqlite::params![ts, head],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let heads = read_bash_heads(&conn, now - 3600);
|
||||||
|
assert_eq!(heads.get("cargo").copied(), Some(2));
|
||||||
|
assert_eq!(heads.get("git").copied(), Some(1));
|
||||||
|
assert_eq!(heads.get("rg").copied(), None); // window cutoff excludes it
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A missing `bash_commands` table degrades to an empty tally rather
|
||||||
|
/// than erroring — the pre-capture window on a fresh agent.
|
||||||
|
#[test]
|
||||||
|
fn bash_heads_missing_table_is_empty() {
|
||||||
|
let conn = Connection::open_in_memory().unwrap();
|
||||||
|
assert!(read_bash_heads(&conn, 0).is_empty());
|
||||||
|
}
|
||||||
|
}
|
||||||
260
hive-c0re/src/stats/host_stats.rs
Normal file
260
hive-c0re/src/stats/host_stats.rs
Normal file
|
|
@ -0,0 +1,260 @@
|
||||||
|
//! Host-system probes and the derived **server-warning** list that the
|
||||||
|
//! dashboard renders as a top-of-page banner.
|
||||||
|
//!
|
||||||
|
//! The first (and currently only) producer is a host disk-usage check:
|
||||||
|
//! the nix store filling up is what ENOSPC-failed CI before the GC
|
||||||
|
//! guardrails were documented. hive-c0re runs on the host (not inside a
|
||||||
|
//! container), so it can `statvfs` the store path directly and warn the
|
||||||
|
//! operator *before* an ENOSPC, not after.
|
||||||
|
//!
|
||||||
|
//! [`server_warnings`] is the public surface: it returns a flat list of
|
||||||
|
//! [`ServerWarning`]s for `/api/state`. The dashboard renders whatever it
|
||||||
|
//! returns, coloured by `level`, so adding a new system warning (memory
|
||||||
|
//! pressure, a failed unit, …) is a backend-only change — no frontend
|
||||||
|
//! edit. Keep producers cheap; this runs on every `/api/state` assembly.
|
||||||
|
|
||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
|
use serde::Serialize;
|
||||||
|
|
||||||
|
use crate::container_view::ContainerView;
|
||||||
|
|
||||||
|
/// One server-level warning for the dashboard's top-of-page banner.
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
pub struct ServerWarning {
|
||||||
|
/// Stable kind id (e.g. `"disk_pressure"`) — lets the frontend dedupe
|
||||||
|
/// or special-case without parsing the message.
|
||||||
|
pub kind: &'static str,
|
||||||
|
/// Severity: `"warn"` (amber) or `"crit"` (red). The banner picks its
|
||||||
|
/// colour from this; everything else is just the message text.
|
||||||
|
pub level: &'static str,
|
||||||
|
/// Human-readable, already-formatted message shown in the banner.
|
||||||
|
pub message: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Percent-used past which the host nix store earns a disk-pressure
|
||||||
|
/// warning; above [`DISK_CRIT_PCT`] it escalates to `crit`.
|
||||||
|
const DISK_WARN_PCT: f64 = 85.0;
|
||||||
|
const DISK_CRIT_PCT: f64 = 95.0;
|
||||||
|
|
||||||
|
/// Collect the current server-level warnings for the dashboard banner.
|
||||||
|
/// Each producer pushes zero or more [`ServerWarning`]s; the frontend
|
||||||
|
/// renders whatever this returns. Cheap to call on every `/api/state`
|
||||||
|
/// assembly (currently a single `statvfs`).
|
||||||
|
#[must_use]
|
||||||
|
pub fn server_warnings() -> Vec<ServerWarning> {
|
||||||
|
let mut out = Vec::new();
|
||||||
|
if let Some(d) = nix_disk_usage()
|
||||||
|
&& d.used_pct >= DISK_WARN_PCT
|
||||||
|
{
|
||||||
|
#[allow(
|
||||||
|
clippy::cast_precision_loss,
|
||||||
|
reason = "byte counts stay well under f64's 2^53 exact-integer range, so this GiB conversion loses no precision"
|
||||||
|
)]
|
||||||
|
let free_gib = d.free_bytes as f64 / (1024.0 * 1024.0 * 1024.0);
|
||||||
|
out.push(ServerWarning {
|
||||||
|
kind: "disk_pressure",
|
||||||
|
level: if d.used_pct >= DISK_CRIT_PCT {
|
||||||
|
"crit"
|
||||||
|
} else {
|
||||||
|
"warn"
|
||||||
|
},
|
||||||
|
message: format!(
|
||||||
|
"host nix store {:.0}% full ({free_gib:.1} GiB free) \
|
||||||
|
— garbage-collect the store before it runs out of space",
|
||||||
|
d.used_pct
|
||||||
|
),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Agent-state warnings derived from the live container snapshot the
|
||||||
|
/// dashboard already holds: agents that need a claude login, and agents
|
||||||
|
/// that are crashing. Kept separate from [`server_warnings`] (host
|
||||||
|
/// probes) because the caller owns the container list + crash counts;
|
||||||
|
/// the dashboard concatenates both into one banner list.
|
||||||
|
///
|
||||||
|
/// `crash_counts` is `Coordinator::recent_crash_counts(window)` — agent →
|
||||||
|
/// number of crashes inside that window — so a crash-looping agent shows
|
||||||
|
/// its repeat count rather than a single point-in-time flap.
|
||||||
|
#[must_use]
|
||||||
|
pub fn agent_state_warnings<S: std::hash::BuildHasher>(
|
||||||
|
containers: &[ContainerView],
|
||||||
|
crash_counts: &HashMap<String, usize, S>,
|
||||||
|
) -> Vec<ServerWarning> {
|
||||||
|
let mut out = Vec::new();
|
||||||
|
|
||||||
|
// `needs_login` is already running-gated in `container_view::build_all`,
|
||||||
|
// so a stopped container never lights this.
|
||||||
|
let mut pending: Vec<&str> = containers
|
||||||
|
.iter()
|
||||||
|
.filter(|c| c.needs_login)
|
||||||
|
.map(|c| c.name.as_str())
|
||||||
|
.collect();
|
||||||
|
if !pending.is_empty() {
|
||||||
|
pending.sort_unstable();
|
||||||
|
out.push(ServerWarning {
|
||||||
|
kind: "pending_logins",
|
||||||
|
level: "warn",
|
||||||
|
message: format!(
|
||||||
|
"{n} agent{plural} {verb} claude login: {list} \
|
||||||
|
— open the agent's page in the dashboard to complete the login",
|
||||||
|
n = pending.len(),
|
||||||
|
plural = if pending.len() == 1 { "" } else { "s" },
|
||||||
|
verb = if pending.len() == 1 { "needs" } else { "need" },
|
||||||
|
list = pending.join(", "),
|
||||||
|
),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if !crash_counts.is_empty() {
|
||||||
|
let mut crashing: Vec<(&String, usize)> =
|
||||||
|
crash_counts.iter().map(|(a, n)| (a, *n)).collect();
|
||||||
|
crashing.sort_unstable_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(b.0)));
|
||||||
|
let list = crashing
|
||||||
|
.iter()
|
||||||
|
.map(|(a, n)| format!("{a} (×{n})"))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", ");
|
||||||
|
out.push(ServerWarning {
|
||||||
|
kind: "agents_crashing",
|
||||||
|
level: "crit",
|
||||||
|
message: format!(
|
||||||
|
"{n} agent{plural} crashing: {list} \
|
||||||
|
— check the container journal (`hivectl logs <agent>`)",
|
||||||
|
n = crashing.len(),
|
||||||
|
plural = if crashing.len() == 1 { "" } else { "s" },
|
||||||
|
),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Disk usage for the filesystem backing the host nix store — internal to
|
||||||
|
/// the disk-pressure producer above.
|
||||||
|
struct DiskUsage {
|
||||||
|
/// Space available to unprivileged writers, in bytes.
|
||||||
|
free_bytes: u64,
|
||||||
|
/// Percentage used, 0–100. Mirrors `df`'s use% — `used / (used +
|
||||||
|
/// available)` — so the threshold means what the operator sees in `df`.
|
||||||
|
used_pct: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Probe disk usage for the filesystem containing the nix store (`/nix`),
|
||||||
|
/// falling back to `/` when `/nix` isn't its own mount. Returns `None` if
|
||||||
|
/// the `statvfs` syscall fails (path missing, permission, etc.).
|
||||||
|
fn nix_disk_usage() -> Option<DiskUsage> {
|
||||||
|
disk_usage("/nix").or_else(|| disk_usage("/"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn disk_usage(path: &str) -> Option<DiskUsage> {
|
||||||
|
let c_path = std::ffi::CString::new(path).ok()?;
|
||||||
|
// SAFETY: `statvfs` reads only through the valid NUL-terminated
|
||||||
|
// `c_path` pointer and writes into the zeroed `stat` we own. We check
|
||||||
|
// the return code before reading any field.
|
||||||
|
let mut stat: libc::statvfs = unsafe { std::mem::zeroed() };
|
||||||
|
let rc = unsafe { libc::statvfs(c_path.as_ptr(), &raw mut stat) };
|
||||||
|
if rc != 0 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
// statvfs fields are `c_ulong` (== u64 on the x86_64-linux host this
|
||||||
|
// runs on); the arithmetic below stays in that native width.
|
||||||
|
let frsize = stat.f_frsize;
|
||||||
|
let total_blocks = stat.f_blocks;
|
||||||
|
let free_blocks = stat.f_bfree;
|
||||||
|
let avail_blocks = stat.f_bavail;
|
||||||
|
if total_blocks == 0 || frsize == 0 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let free_bytes = avail_blocks.saturating_mul(frsize);
|
||||||
|
// df's use%: used / (used + available). `used` counts root-reserved
|
||||||
|
// blocks (total - bfree); `available` is the unprivileged free
|
||||||
|
// (bavail), so the percentage matches what `df` reports.
|
||||||
|
let used_blocks = total_blocks.saturating_sub(free_blocks);
|
||||||
|
let capacity = used_blocks.saturating_add(avail_blocks);
|
||||||
|
let used_pct = if capacity == 0 {
|
||||||
|
0.0
|
||||||
|
} else {
|
||||||
|
#[allow(
|
||||||
|
clippy::cast_precision_loss,
|
||||||
|
reason = "block counts stay well under f64's 2^53 exact-integer range, so this percentage computation loses no precision"
|
||||||
|
)]
|
||||||
|
let raw = used_blocks as f64 / capacity as f64 * 100.0;
|
||||||
|
raw
|
||||||
|
};
|
||||||
|
Some(DiskUsage {
|
||||||
|
free_bytes,
|
||||||
|
used_pct,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn cv(name: &str, needs_login: bool) -> ContainerView {
|
||||||
|
ContainerView {
|
||||||
|
name: name.to_owned(),
|
||||||
|
container: format!("h-{name}"),
|
||||||
|
port: 0,
|
||||||
|
running: true,
|
||||||
|
needs_update: false,
|
||||||
|
needs_login,
|
||||||
|
deployed_sha: None,
|
||||||
|
pending_reminders: 0,
|
||||||
|
parent: None,
|
||||||
|
active_model: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn no_agent_warnings_when_all_healthy() {
|
||||||
|
let containers = [cv("alice", false), cv("bob", false)];
|
||||||
|
assert!(agent_state_warnings(&containers, &HashMap::new()).is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pending_logins_lists_sorted_agents() {
|
||||||
|
let containers = [cv("zoe", true), cv("amy", false), cv("bob", true)];
|
||||||
|
let w = agent_state_warnings(&containers, &HashMap::new());
|
||||||
|
assert_eq!(w.len(), 1);
|
||||||
|
assert_eq!(w[0].kind, "pending_logins");
|
||||||
|
assert_eq!(w[0].level, "warn");
|
||||||
|
// sorted, login-needing only, count reflected
|
||||||
|
assert!(
|
||||||
|
w[0].message
|
||||||
|
.starts_with("2 agents need claude login: bob, zoe")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn singular_grammar_for_one_agent() {
|
||||||
|
let containers = [cv("solo", true)];
|
||||||
|
let w = agent_state_warnings(&containers, &HashMap::new());
|
||||||
|
assert!(w[0].message.starts_with("1 agent needs claude login: solo"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn crashing_warning_is_crit_and_count_ordered() {
|
||||||
|
let crashes = HashMap::from([("flap".to_owned(), 5), ("blip".to_owned(), 1)]);
|
||||||
|
let w = agent_state_warnings(&[], &crashes);
|
||||||
|
assert_eq!(w.len(), 1);
|
||||||
|
assert_eq!(w[0].kind, "agents_crashing");
|
||||||
|
assert_eq!(w[0].level, "crit");
|
||||||
|
// higher crash count first
|
||||||
|
assert!(w[0].message.contains("flap (×5), blip (×1)"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn both_warnings_coexist() {
|
||||||
|
let containers = [cv("a", true)];
|
||||||
|
let crashes = HashMap::from([("b".to_owned(), 2)]);
|
||||||
|
let kinds: Vec<&str> = agent_state_warnings(&containers, &crashes)
|
||||||
|
.iter()
|
||||||
|
.map(|w| w.kind)
|
||||||
|
.collect();
|
||||||
|
assert_eq!(kinds, ["pending_logins", "agents_crashing"]);
|
||||||
|
}
|
||||||
|
}
|
||||||
8
hive-c0re/src/stats/mod.rs
Normal file
8
hive-c0re/src/stats/mod.rs
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
//! Metrics aggregation for the dashboard: hive-wide turn-stats
|
||||||
|
//! rollups, host-system probes / server warnings, and live
|
||||||
|
//! per-container cgroup load. Each submodule is re-exported at the
|
||||||
|
//! crate root, so `crate::hive_stats::…` etc. keep working unchanged.
|
||||||
|
|
||||||
|
pub mod container_stats;
|
||||||
|
pub mod hive_stats;
|
||||||
|
pub mod host_stats;
|
||||||
515
hive-c0re/src/stores/approvals.rs
Normal file
515
hive-c0re/src/stores/approvals.rs
Normal file
|
|
@ -0,0 +1,515 @@
|
||||||
|
//! Approval queue. Manager submits via `RequestApplyCommit`; the user
|
||||||
|
//! approves/denies via the host admin CLI; on approval the host runs the
|
||||||
|
//! corresponding action (Phase 5a: `lifecycle::rebuild(agent)`).
|
||||||
|
|
||||||
|
use std::path::Path;
|
||||||
|
use std::sync::Mutex;
|
||||||
|
|
||||||
|
use anyhow::{Context, Result, bail};
|
||||||
|
use hive_sh4re::wire_time::now_unix;
|
||||||
|
use hive_sh4re::{Approval, ApprovalKind, ApprovalStatus};
|
||||||
|
use rusqlite::{Connection, OptionalExtension, params};
|
||||||
|
|
||||||
|
const SCHEMA: &str = r"
|
||||||
|
CREATE TABLE IF NOT EXISTS approvals (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
agent TEXT NOT NULL,
|
||||||
|
commit_ref TEXT NOT NULL,
|
||||||
|
requested_at INTEGER NOT NULL,
|
||||||
|
status TEXT NOT NULL,
|
||||||
|
resolved_at INTEGER,
|
||||||
|
note TEXT
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_approvals_pending
|
||||||
|
ON approvals (id) WHERE status = 'pending';
|
||||||
|
";
|
||||||
|
|
||||||
|
/// Additive column migrations for pre-existing databases, applied via
|
||||||
|
/// `db::apply_migrations` (try-and-ignore-duplicate-column).
|
||||||
|
const MIGRATIONS: &[&str] = &[
|
||||||
|
// `kind` (pre-Phase-8 dbs): legacy rows default to `apply_commit`,
|
||||||
|
// which matches their actual semantics.
|
||||||
|
"ALTER TABLE approvals ADD COLUMN kind TEXT NOT NULL DEFAULT 'apply_commit'",
|
||||||
|
// `description`: manager-supplied note shown on the dashboard
|
||||||
|
// approval card at submission time (distinct from `note`, set on
|
||||||
|
// denial/failure).
|
||||||
|
"ALTER TABLE approvals ADD COLUMN description TEXT",
|
||||||
|
// `fetched_sha`: the canonical sha hive-c0re vouched for at
|
||||||
|
// `request_apply_commit` time. Distinct from `commit_ref`
|
||||||
|
// (manager-supplied, may not even resolve by approve time).
|
||||||
|
"ALTER TABLE approvals ADD COLUMN fetched_sha TEXT",
|
||||||
|
// `submitter`: the agent that submitted the approval (the
|
||||||
|
// authenticated socket caller); approval-scoped helper events
|
||||||
|
// route to it. Legacy rows are NULL → callers fall back to the
|
||||||
|
// root agent.
|
||||||
|
"ALTER TABLE approvals ADD COLUMN submitter TEXT",
|
||||||
|
];
|
||||||
|
|
||||||
|
pub struct Approvals {
|
||||||
|
conn: Mutex<Connection>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Approvals {
|
||||||
|
pub fn open(path: &Path) -> Result<Self> {
|
||||||
|
let conn = crate::db::open(path, "approvals")?;
|
||||||
|
conn.execute_batch(SCHEMA)
|
||||||
|
.context("apply approvals schema")?;
|
||||||
|
crate::db::apply_migrations(&conn, "approvals", MIGRATIONS)?;
|
||||||
|
Ok(Self {
|
||||||
|
conn: Mutex::new(conn),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn submit_kind(
|
||||||
|
&self,
|
||||||
|
agent: &str,
|
||||||
|
kind: ApprovalKind,
|
||||||
|
commit_ref: &str,
|
||||||
|
description: Option<&str>,
|
||||||
|
submitter: &str,
|
||||||
|
) -> Result<i64> {
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO approvals
|
||||||
|
(agent, kind, commit_ref, requested_at, status, description, submitter)
|
||||||
|
VALUES (?1, ?2, ?3, ?4, 'pending', ?5, ?6)",
|
||||||
|
params![
|
||||||
|
agent,
|
||||||
|
kind.as_str(),
|
||||||
|
commit_ref,
|
||||||
|
now_unix(),
|
||||||
|
description,
|
||||||
|
submitter
|
||||||
|
],
|
||||||
|
)?;
|
||||||
|
Ok(conn.last_insert_rowid())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The agent that submitted approval `id` (the authenticated socket
|
||||||
|
/// caller at submit time). `None` for legacy rows predating the
|
||||||
|
/// `submitter` column — callers route those to the root agent.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
///
|
||||||
|
/// Returns an error if the sqlite prepare/query fails. A missing row
|
||||||
|
/// or a `NULL` submitter is not an error — both yield `Ok(None)`.
|
||||||
|
pub fn submitter_of(&self, id: i64) -> Result<Option<String>> {
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
let submitter: Option<String> = conn
|
||||||
|
.query_row(
|
||||||
|
"SELECT submitter FROM approvals WHERE id = ?1",
|
||||||
|
params![id],
|
||||||
|
|row| row.get(0),
|
||||||
|
)
|
||||||
|
.optional()?
|
||||||
|
.flatten();
|
||||||
|
Ok(submitter)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record the canonical sha hive-c0re fetched from the proposed repo
|
||||||
|
/// into applied at submission time. Idempotent on identical values.
|
||||||
|
pub fn set_fetched_sha(&self, id: i64, sha: &str) -> Result<()> {
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE approvals SET fetched_sha = ?1 WHERE id = ?2",
|
||||||
|
params![sha, id],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Last `limit` resolved approvals (approved / denied / failed),
|
||||||
|
/// newest-first. Drives the history tab on the dashboard.
|
||||||
|
pub fn recent_resolved(&self, limit: u64) -> Result<Vec<Approval>> {
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
let mut stmt = conn.prepare(
|
||||||
|
"SELECT id, agent, kind, commit_ref, requested_at, status, resolved_at, note, fetched_sha, description
|
||||||
|
FROM approvals
|
||||||
|
WHERE status IN ('approved', 'denied', 'failed', 'cancelled')
|
||||||
|
ORDER BY resolved_at DESC, id DESC
|
||||||
|
LIMIT ?1",
|
||||||
|
)?;
|
||||||
|
let rows = stmt.query_map([limit], row_to_approval)?;
|
||||||
|
Ok(collect_lenient(rows))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn pending(&self) -> Result<Vec<Approval>> {
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
let mut stmt = conn.prepare(
|
||||||
|
"SELECT id, agent, kind, commit_ref, requested_at, status, resolved_at, note, fetched_sha, description
|
||||||
|
FROM approvals
|
||||||
|
WHERE status = 'pending'
|
||||||
|
ORDER BY id ASC",
|
||||||
|
)?;
|
||||||
|
let rows = stmt.query_map([], row_to_approval)?;
|
||||||
|
Ok(collect_lenient(rows))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get(&self, id: i64) -> Result<Option<Approval>> {
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
conn.query_row(
|
||||||
|
"SELECT id, agent, kind, commit_ref, requested_at, status, resolved_at, note, fetched_sha, description
|
||||||
|
FROM approvals WHERE id = ?1",
|
||||||
|
params![id],
|
||||||
|
row_to_approval,
|
||||||
|
)
|
||||||
|
.optional()
|
||||||
|
.map_err(Into::into)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Mark pending -> approved (or fail if not pending). Returns the (now-updated)
|
||||||
|
/// approval so the caller can run the action and pass the agent name.
|
||||||
|
pub fn mark_approved(&self, id: i64) -> Result<Approval> {
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
let row: Option<ApprovalLookup> = conn
|
||||||
|
.query_row(
|
||||||
|
ApprovalLookup::SELECT,
|
||||||
|
params![id],
|
||||||
|
ApprovalLookup::from_row,
|
||||||
|
)
|
||||||
|
.optional()?;
|
||||||
|
let Some(row) = row else {
|
||||||
|
bail!("approval {id} not found");
|
||||||
|
};
|
||||||
|
if row.status != "pending" {
|
||||||
|
bail!("approval {id} is {}, not pending", row.status);
|
||||||
|
}
|
||||||
|
let resolved_at = now_unix();
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE approvals SET status = 'approved', resolved_at = ?1 WHERE id = ?2",
|
||||||
|
params![resolved_at, id],
|
||||||
|
)?;
|
||||||
|
Ok(Approval {
|
||||||
|
id,
|
||||||
|
agent: row.agent,
|
||||||
|
kind: kind_from_str(&row.kind)?,
|
||||||
|
commit_ref: row.commit_ref,
|
||||||
|
requested_at: hive_sh4re::wire_time::from_secs(row.requested_at),
|
||||||
|
status: ApprovalStatus::Approved,
|
||||||
|
resolved_at: Some(hive_sh4re::wire_time::from_secs(resolved_at)),
|
||||||
|
note: None,
|
||||||
|
fetched_sha: row.fetched_sha,
|
||||||
|
description: row.description,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn mark_denied(&self, id: i64, note: Option<&str>) -> Result<()> {
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
let affected = conn.execute(
|
||||||
|
"UPDATE approvals SET status = 'denied', resolved_at = ?1, note = ?2
|
||||||
|
WHERE id = ?3 AND status = 'pending'",
|
||||||
|
params![now_unix(), note, id],
|
||||||
|
)?;
|
||||||
|
if affected == 0 {
|
||||||
|
bail!("approval {id} not pending");
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn mark_failed(&self, id: i64, note: &str) -> Result<()> {
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE approvals SET status = 'failed', resolved_at = ?1, note = ?2 WHERE id = ?3",
|
||||||
|
params![now_unix(), note, id],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Withdraw a pending approval. Returns the now-updated
|
||||||
|
/// row so the caller can emit `ApprovalResolved` with the right
|
||||||
|
/// kind / agent / sha. Errors if the approval isn't pending — once
|
||||||
|
/// it's approved/denied/failed/cancelled, the resolution is final.
|
||||||
|
pub fn mark_cancelled(&self, id: i64, canceller: &str) -> Result<Approval> {
|
||||||
|
let mut conn = self.conn.lock().unwrap();
|
||||||
|
let tx = conn.transaction()?;
|
||||||
|
let row: Option<ApprovalLookup> = tx
|
||||||
|
.query_row(
|
||||||
|
ApprovalLookup::SELECT,
|
||||||
|
params![id],
|
||||||
|
ApprovalLookup::from_row,
|
||||||
|
)
|
||||||
|
.optional()?;
|
||||||
|
let Some(row) = row else {
|
||||||
|
bail!("approval {id} not found");
|
||||||
|
};
|
||||||
|
if row.status != "pending" {
|
||||||
|
bail!("approval {id} is {}, not pending", row.status);
|
||||||
|
}
|
||||||
|
let resolved_at = now_unix();
|
||||||
|
let note = format!("cancelled by {canceller}");
|
||||||
|
tx.execute(
|
||||||
|
"UPDATE approvals SET status = 'cancelled', resolved_at = ?1, note = ?2 WHERE id = ?3",
|
||||||
|
params![resolved_at, note, id],
|
||||||
|
)?;
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(Approval {
|
||||||
|
id,
|
||||||
|
agent: row.agent,
|
||||||
|
kind: kind_from_str(&row.kind)?,
|
||||||
|
commit_ref: row.commit_ref,
|
||||||
|
requested_at: hive_sh4re::wire_time::from_secs(row.requested_at),
|
||||||
|
status: ApprovalStatus::Cancelled,
|
||||||
|
resolved_at: Some(hive_sh4re::wire_time::from_secs(resolved_at)),
|
||||||
|
note: Some(note),
|
||||||
|
fetched_sha: row.fetched_sha,
|
||||||
|
description: row.description,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Mark every pending approval for `agent` as failed (returns rows affected).
|
||||||
|
/// Used by `destroy` to clear the queue of an agent that no longer exists.
|
||||||
|
pub fn fail_pending_for_agent(&self, agent: &str, note: &str) -> Result<usize> {
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
let n = conn.execute(
|
||||||
|
"UPDATE approvals SET status = 'failed', resolved_at = ?1, note = ?2
|
||||||
|
WHERE agent = ?3 AND status = 'pending'",
|
||||||
|
params![now_unix(), note, agent],
|
||||||
|
)?;
|
||||||
|
Ok(n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Columns needed to rebuild an [`Approval`] after a status transition,
|
||||||
|
/// shared by `mark_approved` / `mark_cancelled`. Replaces a 7-field
|
||||||
|
/// tuple that tripped `clippy::type_complexity` and was duplicated
|
||||||
|
/// across both callers (one suppressed the lint, the other aliased the
|
||||||
|
/// tuple) — one named projection + mapper now backs both.
|
||||||
|
struct ApprovalLookup {
|
||||||
|
agent: String,
|
||||||
|
kind: String,
|
||||||
|
commit_ref: String,
|
||||||
|
requested_at: i64,
|
||||||
|
status: String,
|
||||||
|
fetched_sha: Option<String>,
|
||||||
|
description: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ApprovalLookup {
|
||||||
|
/// The single-row lookup by id (`?1`). Column order matches
|
||||||
|
/// [`ApprovalLookup::from_row`].
|
||||||
|
const SELECT: &str = "SELECT agent, kind, commit_ref, requested_at, status, fetched_sha, \
|
||||||
|
description FROM approvals WHERE id = ?1";
|
||||||
|
|
||||||
|
fn from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<Self> {
|
||||||
|
Ok(Self {
|
||||||
|
agent: row.get(0)?,
|
||||||
|
kind: row.get(1)?,
|
||||||
|
commit_ref: row.get(2)?,
|
||||||
|
requested_at: row.get(3)?,
|
||||||
|
status: row.get(4)?,
|
||||||
|
fetched_sha: row.get(5)?,
|
||||||
|
description: row.get(6)?,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Collect approval rows, dropping (and logging) any that fail to
|
||||||
|
/// deserialize. A single malformed / unknown-kind row must never blank
|
||||||
|
/// the whole list: `collect::<Result<Vec>>()` is all-or-nothing, so one
|
||||||
|
/// bad row used to make `pending()` / `recent_resolved()` error out
|
||||||
|
/// wholesale — the dashboard then rendered an empty approvals queue.
|
||||||
|
fn collect_lenient(rows: impl Iterator<Item = rusqlite::Result<Approval>>) -> Vec<Approval> {
|
||||||
|
rows.filter_map(|r| match r {
|
||||||
|
Ok(a) => Some(a),
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!(error = ?e, "skipping unparseable approval row");
|
||||||
|
None
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn row_to_approval(row: &rusqlite::Row<'_>) -> rusqlite::Result<Approval> {
|
||||||
|
// Column order: id, agent, kind, commit_ref, requested_at, status, resolved_at, note, fetched_sha, description.
|
||||||
|
let kind: String = row.get(2)?;
|
||||||
|
let kind = match kind.as_str() {
|
||||||
|
"apply_commit" => ApprovalKind::ApplyCommit,
|
||||||
|
"spawn" => ApprovalKind::Spawn,
|
||||||
|
"init_config" => ApprovalKind::InitConfig,
|
||||||
|
"update_meta_inputs" => ApprovalKind::UpdateMetaInputs,
|
||||||
|
"schedule_prompt" => ApprovalKind::SchedulePrompt,
|
||||||
|
"merge_config_pr" => ApprovalKind::MergeConfigPr,
|
||||||
|
other => {
|
||||||
|
return Err(rusqlite::Error::FromSqlConversionFailure(
|
||||||
|
2,
|
||||||
|
rusqlite::types::Type::Text,
|
||||||
|
format!("unknown approval kind '{other}'").into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let status: String = row.get(5)?;
|
||||||
|
let status = match status.as_str() {
|
||||||
|
"pending" => ApprovalStatus::Pending,
|
||||||
|
"approved" => ApprovalStatus::Approved,
|
||||||
|
"denied" => ApprovalStatus::Denied,
|
||||||
|
"failed" => ApprovalStatus::Failed,
|
||||||
|
"cancelled" => ApprovalStatus::Cancelled,
|
||||||
|
other => {
|
||||||
|
return Err(rusqlite::Error::FromSqlConversionFailure(
|
||||||
|
5,
|
||||||
|
rusqlite::types::Type::Text,
|
||||||
|
format!("unknown approval status '{other}'").into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
Ok(Approval {
|
||||||
|
id: row.get(0)?,
|
||||||
|
agent: row.get(1)?,
|
||||||
|
kind,
|
||||||
|
commit_ref: row.get(3)?,
|
||||||
|
requested_at: hive_sh4re::wire_time::from_secs(row.get(4)?),
|
||||||
|
status,
|
||||||
|
resolved_at: row
|
||||||
|
.get::<_, Option<i64>>(6)?
|
||||||
|
.map(hive_sh4re::wire_time::from_secs),
|
||||||
|
note: row.get(7)?,
|
||||||
|
fetched_sha: row.get(8)?,
|
||||||
|
description: row.get(9)?,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn kind_from_str(s: &str) -> Result<ApprovalKind> {
|
||||||
|
Ok(match s {
|
||||||
|
"apply_commit" => ApprovalKind::ApplyCommit,
|
||||||
|
"spawn" => ApprovalKind::Spawn,
|
||||||
|
"init_config" => ApprovalKind::InitConfig,
|
||||||
|
"update_meta_inputs" => ApprovalKind::UpdateMetaInputs,
|
||||||
|
"schedule_prompt" => ApprovalKind::SchedulePrompt,
|
||||||
|
"merge_config_pr" => ApprovalKind::MergeConfigPr,
|
||||||
|
other => bail!("unknown approval kind '{other}'"),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use hive_sh4re::ApprovalKind;
|
||||||
|
|
||||||
|
fn open_temp() -> (tempfile::TempDir, std::path::PathBuf, Approvals) {
|
||||||
|
let dir = tempfile::tempdir().expect("tempdir");
|
||||||
|
let path = dir.path().join("approvals.sqlite");
|
||||||
|
let db = Approvals::open(&path).expect("open approvals db");
|
||||||
|
(dir, path, db)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn init_config_approval_round_trips() {
|
||||||
|
// Regression test: an `init_config` row used to fail
|
||||||
|
// deserialization (row_to_approval matched only apply_commit +
|
||||||
|
// spawn), erroring out the whole `pending()` query — every
|
||||||
|
// approval then vanished from the dashboard.
|
||||||
|
let (_dir, _path, db) = open_temp();
|
||||||
|
let id = db
|
||||||
|
.submit_kind(
|
||||||
|
"bitburner",
|
||||||
|
ApprovalKind::InitConfig,
|
||||||
|
"",
|
||||||
|
Some("scaffold"),
|
||||||
|
"bitburner",
|
||||||
|
)
|
||||||
|
.expect("submit init_config");
|
||||||
|
let pending = db
|
||||||
|
.pending()
|
||||||
|
.expect("pending() must not error on an init_config row");
|
||||||
|
assert_eq!(pending.len(), 1);
|
||||||
|
assert_eq!(pending[0].id, id);
|
||||||
|
assert!(matches!(pending[0].kind, ApprovalKind::InitConfig));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn mixed_kinds_all_listed() {
|
||||||
|
let (_dir, _path, db) = open_temp();
|
||||||
|
db.submit_kind("a", ApprovalKind::ApplyCommit, "deadbeef", None, "a")
|
||||||
|
.unwrap();
|
||||||
|
db.submit_kind("b", ApprovalKind::Spawn, "", None, "b")
|
||||||
|
.unwrap();
|
||||||
|
db.submit_kind("c", ApprovalKind::InitConfig, "", None, "c")
|
||||||
|
.unwrap();
|
||||||
|
let pending = db.pending().expect("pending");
|
||||||
|
assert_eq!(pending.len(), 3, "all three kinds must be visible");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn mark_cancelled_transitions_pending_row() {
|
||||||
|
// Manager withdraws a pending approval. Row leaves pending(),
|
||||||
|
// shows up in recent_resolved() with the cancelled status + a
|
||||||
|
// "cancelled by <who>" note.
|
||||||
|
let (_dir, _path, db) = open_temp();
|
||||||
|
let id = db
|
||||||
|
.submit_kind(
|
||||||
|
"bitburner",
|
||||||
|
ApprovalKind::ApplyCommit,
|
||||||
|
"cafef00d",
|
||||||
|
Some("test"),
|
||||||
|
"bitburner",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let row = db.mark_cancelled(id, "manager").expect("cancel");
|
||||||
|
assert_eq!(row.id, id);
|
||||||
|
assert!(matches!(row.status, ApprovalStatus::Cancelled));
|
||||||
|
assert_eq!(row.note.as_deref(), Some("cancelled by manager"));
|
||||||
|
assert!(row.resolved_at.is_some());
|
||||||
|
assert!(db.pending().unwrap().is_empty(), "row leaves pending");
|
||||||
|
let resolved = db.recent_resolved(10).unwrap();
|
||||||
|
assert_eq!(resolved.len(), 1);
|
||||||
|
assert!(matches!(resolved[0].status, ApprovalStatus::Cancelled));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn mark_cancelled_refuses_already_resolved_row() {
|
||||||
|
// Once approved/denied/failed/cancelled the resolution is
|
||||||
|
// final — re-cancelling errors instead of silently overwriting.
|
||||||
|
let (_dir, _path, db) = open_temp();
|
||||||
|
let id = db
|
||||||
|
.submit_kind("a", ApprovalKind::Spawn, "deadbeef", None, "a")
|
||||||
|
.unwrap();
|
||||||
|
db.mark_cancelled(id, "manager").expect("first cancel");
|
||||||
|
let err = db
|
||||||
|
.mark_cancelled(id, "manager")
|
||||||
|
.expect_err("second cancel must fail");
|
||||||
|
assert!(err.to_string().contains("not pending"), "{err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unknown_kind_row_is_skipped_not_fatal() {
|
||||||
|
// A single malformed / future-kind row must not blank the
|
||||||
|
// whole list — collect_lenient skips it instead of failing.
|
||||||
|
let (_dir, path, db) = open_temp();
|
||||||
|
let good = db
|
||||||
|
.submit_kind("good", ApprovalKind::ApplyCommit, "cafe", None, "good")
|
||||||
|
.unwrap();
|
||||||
|
let raw = Connection::open(&path).unwrap();
|
||||||
|
raw.execute(
|
||||||
|
"INSERT INTO approvals (agent, kind, commit_ref, requested_at, status)
|
||||||
|
VALUES ('weird', 'from_the_future', '', 0, 'pending')",
|
||||||
|
[],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let pending = db
|
||||||
|
.pending()
|
||||||
|
.expect("pending() must survive an unparseable row");
|
||||||
|
assert_eq!(pending.len(), 1);
|
||||||
|
assert_eq!(pending[0].id, good);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn submitter_round_trips_and_legacy_reads_none() {
|
||||||
|
// A submitted approval records its submitter; a legacy row
|
||||||
|
// (inserted without the column) reads back as None so callers
|
||||||
|
// fall back to the root agent.
|
||||||
|
let (_dir, path, db) = open_temp();
|
||||||
|
let id = db
|
||||||
|
.submit_kind("child", ApprovalKind::ApplyCommit, "cafe", None, "parent")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(db.submitter_of(id).unwrap().as_deref(), Some("parent"));
|
||||||
|
|
||||||
|
let raw = Connection::open(&path).unwrap();
|
||||||
|
raw.execute(
|
||||||
|
"INSERT INTO approvals (agent, kind, commit_ref, requested_at, status)
|
||||||
|
VALUES ('old', 'apply_commit', '', 0, 'pending')",
|
||||||
|
[],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let legacy_id = raw.last_insert_rowid();
|
||||||
|
assert_eq!(db.submitter_of(legacy_id).unwrap(), None);
|
||||||
|
}
|
||||||
|
}
|
||||||
332
hive-c0re/src/stores/audit_log.rs
Normal file
332
hive-c0re/src/stores/audit_log.rs
Normal file
|
|
@ -0,0 +1,332 @@
|
||||||
|
//! Sqlite-backed audit trail of agent-initiated privileged actions.
|
||||||
|
//!
|
||||||
|
//! Surfaces, durably and operator-visibly, the privileged operations
|
||||||
|
//! hive-c0re performs *on behalf of an agent* — the ones that cross the
|
||||||
|
//! agent/operator trust boundary and so warrant a who/what/when record
|
||||||
|
//! beyond hive-priv's low-level journal trace. First entry: infra
|
||||||
|
//! container restarts via the `infra_admin`-gated `restart` tool (the
|
||||||
|
//! follow-up audit trail for that capability).
|
||||||
|
//!
|
||||||
|
//! Deliberately scoped to *agent-initiated* privileged actions. The bulk
|
||||||
|
//! of `PrivRequest` traffic (token writes, nspawn-flag edits) fires
|
||||||
|
//! constantly during normal lifecycle and is hive-c0re's own bookkeeping,
|
||||||
|
//! not an agent crossing the boundary — logging all of it would drown the
|
||||||
|
//! signal the operator actually wants.
|
||||||
|
//!
|
||||||
|
//! Same process-singleton handle pattern as `build_logs`: installed once
|
||||||
|
//! at `Coordinator::open`, fetched via [`global`] so the recording sites
|
||||||
|
//! (e.g. `socket_server::handle_restart_infra`) don't have to thread an
|
||||||
|
//! `Arc<AuditLog>` through every call path. Recording is best-effort: a
|
||||||
|
//! sqlite blip must never fail the underlying privileged action.
|
||||||
|
|
||||||
|
use std::path::Path;
|
||||||
|
use std::sync::{Arc, Mutex, OnceLock};
|
||||||
|
|
||||||
|
use anyhow::{Context, Result};
|
||||||
|
|
||||||
|
use chrono::{DateTime, Utc};
|
||||||
|
use hive_sh4re::wire_time::now_unix;
|
||||||
|
use rusqlite::{Connection, params};
|
||||||
|
use serde::Serialize;
|
||||||
|
|
||||||
|
/// Process-singleton handle, set once at coordinator startup. Mirrors
|
||||||
|
/// `build_logs::GLOBAL` — lets recording sites write without threading an
|
||||||
|
/// `Arc<AuditLog>` through every entry point.
|
||||||
|
static GLOBAL: OnceLock<Arc<AuditLog>> = OnceLock::new();
|
||||||
|
|
||||||
|
/// Install the process-wide `AuditLog` handle. Idempotent: a second call
|
||||||
|
/// silently keeps the first handle.
|
||||||
|
pub fn install(handle: Arc<AuditLog>) {
|
||||||
|
let _ = GLOBAL.set(handle);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fetch the process-wide handle, or `None` if `install` hasn't run yet
|
||||||
|
/// (early startup, or unit tests). Callers must gracefully no-op on `None`.
|
||||||
|
#[must_use]
|
||||||
|
pub fn global() -> Option<Arc<AuditLog>> {
|
||||||
|
GLOBAL.get().cloned()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Retain audit rows for 90 days. Longer than build-log retention — this
|
||||||
|
/// is a security/accountability record, not debug noise; the operator may
|
||||||
|
/// want to review "who restarted what" well after the fact.
|
||||||
|
const KEEP_SECS: i64 = 90 * 24 * 3600;
|
||||||
|
|
||||||
|
const SCHEMA: &str = "
|
||||||
|
CREATE TABLE IF NOT EXISTS audit_log (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
ts_unix INTEGER NOT NULL,
|
||||||
|
agent TEXT NOT NULL,
|
||||||
|
action TEXT NOT NULL,
|
||||||
|
target TEXT NOT NULL,
|
||||||
|
outcome TEXT NOT NULL,
|
||||||
|
detail TEXT
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_audit_log_ts ON audit_log (ts_unix DESC);
|
||||||
|
";
|
||||||
|
|
||||||
|
/// Outcome of a recorded privileged action. Stored as the literal string
|
||||||
|
/// in the `outcome` column.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum AuditOutcome {
|
||||||
|
/// The privileged action succeeded.
|
||||||
|
Ok,
|
||||||
|
/// The privileged action was attempted but failed (e.g. the
|
||||||
|
/// underlying systemctl call errored). Denied-by-capability attempts
|
||||||
|
/// are recorded too — see the recording site.
|
||||||
|
Err,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AuditOutcome {
|
||||||
|
fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Ok => "ok",
|
||||||
|
Self::Err => "err",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One audit row as returned to the dashboard.
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
pub struct AuditEntry {
|
||||||
|
pub id: i64,
|
||||||
|
pub ts_unix: DateTime<Utc>,
|
||||||
|
/// Agent on whose behalf the action was taken.
|
||||||
|
pub agent: String,
|
||||||
|
/// What was done (e.g. `restart_infra`).
|
||||||
|
pub action: String,
|
||||||
|
/// What it acted on (e.g. `hive-ci`).
|
||||||
|
pub target: String,
|
||||||
|
/// `"ok"` | `"err"`.
|
||||||
|
pub outcome: String,
|
||||||
|
/// Optional free-text detail (e.g. the error message on failure).
|
||||||
|
pub detail: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Sqlite-backed audit-log store. `Arc<AuditLog>`-friendly: all methods
|
||||||
|
/// take `&self`, an internal `Mutex<Connection>` serializes access.
|
||||||
|
pub struct AuditLog {
|
||||||
|
conn: Mutex<Connection>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AuditLog {
|
||||||
|
/// Open (creating if absent) the `audit_log.sqlite` store under
|
||||||
|
/// `db_dir` and apply the schema. `db_dir` is shared with
|
||||||
|
/// `build_logs` (the broker db's parent directory).
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Returns an error if the directory can't be created, the sqlite
|
||||||
|
/// file can't be opened, or applying the schema fails.
|
||||||
|
pub fn open(db_dir: &Path) -> Result<Self> {
|
||||||
|
let path = db_dir.join("audit_log.sqlite");
|
||||||
|
let conn = crate::db::open(&path, "audit_log")?;
|
||||||
|
conn.execute_batch(SCHEMA)
|
||||||
|
.context("apply audit_log schema")?;
|
||||||
|
Ok(Self {
|
||||||
|
conn: Mutex::new(conn),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record one privileged action. Best-effort: a sqlite error is logged
|
||||||
|
/// but never returned, so a transient blip never fails the underlying
|
||||||
|
/// privileged action (the action already happened — losing its audit
|
||||||
|
/// row is strictly less bad than failing the action retroactively).
|
||||||
|
///
|
||||||
|
/// Returns the inserted [`AuditEntry`] (with its assigned id +
|
||||||
|
/// timestamp) on success, or `None` if the insert failed. The
|
||||||
|
/// returned row is the canonical record — callers that also push a
|
||||||
|
/// live event (e.g. the dashboard stream) emit *this* rather than
|
||||||
|
/// re-deriving the fields, so the stored row and the streamed event
|
||||||
|
/// can't drift.
|
||||||
|
#[must_use]
|
||||||
|
pub fn record(
|
||||||
|
&self,
|
||||||
|
agent: &str,
|
||||||
|
action: &str,
|
||||||
|
target: &str,
|
||||||
|
outcome: AuditOutcome,
|
||||||
|
detail: Option<&str>,
|
||||||
|
) -> Option<AuditEntry> {
|
||||||
|
let now = now_unix();
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
match conn.execute(
|
||||||
|
"INSERT INTO audit_log (ts_unix, agent, action, target, outcome, detail)
|
||||||
|
VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
||||||
|
params![now, agent, action, target, outcome.as_str(), detail],
|
||||||
|
) {
|
||||||
|
Ok(_) => Some(AuditEntry {
|
||||||
|
id: conn.last_insert_rowid(),
|
||||||
|
ts_unix: hive_sh4re::wire_time::from_secs(now),
|
||||||
|
agent: agent.to_owned(),
|
||||||
|
action: action.to_owned(),
|
||||||
|
target: target.to_owned(),
|
||||||
|
outcome: outcome.as_str().to_owned(),
|
||||||
|
detail: detail.map(str::to_owned),
|
||||||
|
}),
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!(
|
||||||
|
%agent, %action, %target,
|
||||||
|
error = ?e,
|
||||||
|
"audit_log: record failed (dropping entry)"
|
||||||
|
);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return the most recent `limit` rows, newest first. Limit is
|
||||||
|
/// hard-clamped to 500 to bound the worst-case payload.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Returns an error if the query fails to prepare or a row fails to
|
||||||
|
/// deserialize.
|
||||||
|
pub fn list_recent(&self, limit: usize) -> Result<Vec<AuditEntry>> {
|
||||||
|
let limit = limit.min(500);
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
let mut stmt = conn.prepare(
|
||||||
|
"SELECT id, ts_unix, agent, action, target, outcome, detail
|
||||||
|
FROM audit_log
|
||||||
|
ORDER BY ts_unix DESC, id DESC
|
||||||
|
LIMIT ?1",
|
||||||
|
)?;
|
||||||
|
let rows = stmt.query_map(params![i64::try_from(limit).unwrap_or(500)], row_to_entry)?;
|
||||||
|
let mut out = Vec::new();
|
||||||
|
for r in rows {
|
||||||
|
out.push(r?);
|
||||||
|
}
|
||||||
|
Ok(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Total row count, regardless of the `list_recent` clamp. Lets the
|
||||||
|
/// dashboard show "latest N of TOTAL" instead of silently capping.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Returns an error if the `COUNT(*)` query fails.
|
||||||
|
pub fn count_total(&self) -> Result<i64> {
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
let n: i64 = conn.query_row("SELECT COUNT(*) FROM audit_log", [], |r| r.get(0))?;
|
||||||
|
Ok(n)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Drop rows older than the retention window. Returns the number of
|
||||||
|
/// rows deleted. Called from the hourly vacuum loop.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Returns an error if the `DELETE` query fails.
|
||||||
|
pub fn vacuum(&self) -> Result<u64> {
|
||||||
|
let cutoff = now_unix() - KEEP_SECS;
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
let removed = conn.execute("DELETE FROM audit_log WHERE ts_unix < ?1", params![cutoff])?;
|
||||||
|
Ok(u64::try_from(removed).unwrap_or(0))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Spawn the hourly retention sweep. Mirrors `build_logs::spawn_vacuum`
|
||||||
|
/// in cadence + shutdown handling.
|
||||||
|
pub fn spawn_vacuum(coord: &Arc<crate::coordinator::Coordinator>) {
|
||||||
|
use std::time::Duration;
|
||||||
|
let audit = coord.audit_log.clone();
|
||||||
|
let mut shutdown = coord.shutdown_rx();
|
||||||
|
let interval = Duration::from_hours(1);
|
||||||
|
tokio::spawn(async move {
|
||||||
|
loop {
|
||||||
|
match audit.vacuum() {
|
||||||
|
Ok(0) => {}
|
||||||
|
Ok(n) => tracing::info!(removed = n, "audit_log vacuum"),
|
||||||
|
Err(e) => tracing::warn!(error = ?e, "audit_log vacuum failed"),
|
||||||
|
}
|
||||||
|
tokio::select! {
|
||||||
|
() = tokio::time::sleep(interval) => {}
|
||||||
|
_ = shutdown.changed() => {
|
||||||
|
tracing::info!("audit_log vacuum: shutdown signal received");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
fn row_to_entry(r: &rusqlite::Row) -> rusqlite::Result<AuditEntry> {
|
||||||
|
Ok(AuditEntry {
|
||||||
|
id: r.get(0)?,
|
||||||
|
ts_unix: hive_sh4re::wire_time::from_secs(r.get(1)?),
|
||||||
|
agent: r.get(2)?,
|
||||||
|
action: r.get(3)?,
|
||||||
|
target: r.get(4)?,
|
||||||
|
outcome: r.get(5)?,
|
||||||
|
detail: r.get(6)?,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn tmpdb() -> (tempfile::TempDir, AuditLog) {
|
||||||
|
let dir = tempfile::tempdir().expect("tempdir");
|
||||||
|
let db = AuditLog::open(dir.path()).expect("open");
|
||||||
|
(dir, db)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn record_and_list_newest_first() {
|
||||||
|
let (_d, db) = tmpdb();
|
||||||
|
// record() returns the canonical inserted row (id + ts assigned).
|
||||||
|
let entry = db
|
||||||
|
.record("atlas", "restart_infra", "hive-ci", AuditOutcome::Ok, None)
|
||||||
|
.expect("record returns the inserted entry");
|
||||||
|
assert!(entry.id > 0);
|
||||||
|
assert_eq!(entry.target, "hive-ci");
|
||||||
|
assert_eq!(entry.outcome, "ok");
|
||||||
|
assert!(entry.detail.is_none());
|
||||||
|
let _ = db.record(
|
||||||
|
"atlas",
|
||||||
|
"restart_infra",
|
||||||
|
"hive-gateway",
|
||||||
|
AuditOutcome::Err,
|
||||||
|
Some("systemctl failed"),
|
||||||
|
);
|
||||||
|
let rows = db.list_recent(10).expect("list");
|
||||||
|
assert_eq!(rows.len(), 2);
|
||||||
|
// Newest first: the gateway/err row was inserted last.
|
||||||
|
assert_eq!(rows[0].target, "hive-gateway");
|
||||||
|
assert_eq!(rows[0].outcome, "err");
|
||||||
|
assert_eq!(rows[0].detail.as_deref(), Some("systemctl failed"));
|
||||||
|
assert_eq!(rows[1].target, "hive-ci");
|
||||||
|
assert_eq!(rows[1].outcome, "ok");
|
||||||
|
assert!(rows[1].detail.is_none());
|
||||||
|
assert_eq!(rows[0].agent, "atlas");
|
||||||
|
assert_eq!(rows[0].action, "restart_infra");
|
||||||
|
assert_eq!(db.count_total().expect("count"), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn list_clamps_to_500() {
|
||||||
|
let (_d, db) = tmpdb();
|
||||||
|
let _ = db.record("a", "x", "t", AuditOutcome::Ok, None);
|
||||||
|
let rows = db.list_recent(999_999).expect("list");
|
||||||
|
assert!(rows.len() <= 500);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn vacuum_drops_only_old_rows() {
|
||||||
|
let (_d, db) = tmpdb();
|
||||||
|
let _ = db.record("a", "restart_infra", "hive-ci", AuditOutcome::Ok, None);
|
||||||
|
// Backdate it past the retention window.
|
||||||
|
{
|
||||||
|
let conn = db.conn.lock().unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE audit_log SET ts_unix = ?1",
|
||||||
|
params![now_unix() - KEEP_SECS - 60],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let _ = db.record("a", "restart_infra", "hive-forge", AuditOutcome::Ok, None);
|
||||||
|
let removed = db.vacuum().expect("vacuum");
|
||||||
|
assert_eq!(removed, 1, "only the backdated row should be reaped");
|
||||||
|
let rows = db.list_recent(10).expect("list");
|
||||||
|
assert_eq!(rows.len(), 1);
|
||||||
|
assert_eq!(rows[0].target, "hive-forge");
|
||||||
|
}
|
||||||
|
}
|
||||||
1578
hive-c0re/src/stores/broker.rs
Normal file
1578
hive-c0re/src/stores/broker.rs
Normal file
File diff suppressed because it is too large
Load diff
565
hive-c0re/src/stores/build_logs.rs
Normal file
565
hive-c0re/src/stores/build_logs.rs
Normal file
|
|
@ -0,0 +1,565 @@
|
||||||
|
//! Sqlite-backed full build-log capture — stdout + stderr per
|
||||||
|
//! `nixos-container` / `nix build` invocation, accumulated live.
|
||||||
|
//! Schema, indices, retention, and the rationale for replacing
|
||||||
|
//! the old ring buffer: `docs/persistence.md::/var/lib/hyperhive/db/build_logs.sqlite`.
|
||||||
|
|
||||||
|
use std::path::Path;
|
||||||
|
use std::sync::{Arc, Mutex, OnceLock};
|
||||||
|
|
||||||
|
use anyhow::{Context, Result};
|
||||||
|
use hive_sh4re::wire_time::now_unix;
|
||||||
|
use rusqlite::{Connection, OptionalExtension, params};
|
||||||
|
use serde::Serialize;
|
||||||
|
use tokio::sync::broadcast;
|
||||||
|
|
||||||
|
/// Process-singleton handle, set once at coordinator startup. Lets
|
||||||
|
/// the `lifecycle` module's `run` / `prebuild_toplevel` access the
|
||||||
|
/// writer without threading an `Arc<BuildLogs>` through every
|
||||||
|
/// `pub async fn` entry point in the lifecycle surface — there are
|
||||||
|
/// 10+ callsites and the handle is the same `Arc` everywhere
|
||||||
|
/// anyway. Set by `Coordinator::open`.
|
||||||
|
static GLOBAL: OnceLock<Arc<BuildLogs>> = OnceLock::new();
|
||||||
|
|
||||||
|
/// Install the process-wide `BuildLogs` handle. Idempotent: a second
|
||||||
|
/// call (e.g. test harness setup) silently keeps the first handle.
|
||||||
|
pub fn install(handle: Arc<BuildLogs>) {
|
||||||
|
let _ = GLOBAL.set(handle);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fetch the process-wide handle, or `None` if `install` hasn't run
|
||||||
|
/// yet (e.g. early in startup before `Coordinator::open`, or in unit
|
||||||
|
/// tests that don't bother with the global). Callers must gracefully
|
||||||
|
/// no-op when this returns `None`.
|
||||||
|
#[must_use]
|
||||||
|
pub fn global() -> Option<Arc<BuildLogs>> {
|
||||||
|
GLOBAL.get().cloned()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Retain failed-build rows for 30 days — they're what the operator
|
||||||
|
/// needs to investigate when diagnosing a regression.
|
||||||
|
const KEEP_FAIL_SECS: i64 = 30 * 24 * 3600;
|
||||||
|
|
||||||
|
/// Retain successful-build rows for 24 hours — useful for diffing
|
||||||
|
/// what changed across a recent rebuild, but past a day the log is
|
||||||
|
/// noise. In-progress rows are never reaped (they have
|
||||||
|
/// `finished_at IS NULL`).
|
||||||
|
const KEEP_OK_SECS: i64 = 24 * 3600;
|
||||||
|
|
||||||
|
const SCHEMA: &str = "
|
||||||
|
CREATE TABLE IF NOT EXISTS build_logs (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
agent TEXT NOT NULL,
|
||||||
|
kind TEXT NOT NULL,
|
||||||
|
cmdline TEXT NOT NULL,
|
||||||
|
started_at INTEGER NOT NULL,
|
||||||
|
finished_at INTEGER,
|
||||||
|
status TEXT,
|
||||||
|
stdout TEXT NOT NULL DEFAULT '',
|
||||||
|
stderr TEXT NOT NULL DEFAULT ''
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_build_logs_agent_started
|
||||||
|
ON build_logs (agent, started_at DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_build_logs_status_finished
|
||||||
|
ON build_logs (status, finished_at)
|
||||||
|
WHERE finished_at IS NOT NULL;
|
||||||
|
";
|
||||||
|
|
||||||
|
/// Status of a finished build attempt. Stored as the literal string in
|
||||||
|
/// the `status` column; `NULL` while the attempt is still in progress.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum BuildStatus {
|
||||||
|
/// Child exited with success.
|
||||||
|
Ok,
|
||||||
|
/// Child exited non-zero (build / eval failure).
|
||||||
|
Fail,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BuildStatus {
|
||||||
|
fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Ok => "ok",
|
||||||
|
Self::Fail => "fail",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Header-only row returned by `list_recent_for_agent`. Carries the
|
||||||
|
/// metadata the dashboard's agent-card chip needs (status + age +
|
||||||
|
/// id-to-open) without the multi-MB stdout/stderr payload.
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
pub struct BuildLogHeader {
|
||||||
|
pub id: i64,
|
||||||
|
pub agent: String,
|
||||||
|
pub kind: String,
|
||||||
|
pub cmdline: String,
|
||||||
|
pub started_at: i64,
|
||||||
|
pub finished_at: Option<i64>,
|
||||||
|
pub status: Option<String>,
|
||||||
|
/// Elapsed seconds from `started_at` to `finished_at`. `None` while
|
||||||
|
/// the build is still in progress.
|
||||||
|
pub runtime_secs: Option<i64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Full row with stdout/stderr text inlined. Returned by `get_full`,
|
||||||
|
/// backs the side-panel viewer's payload.
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
pub struct BuildLogFull {
|
||||||
|
#[serde(flatten)]
|
||||||
|
pub header: BuildLogHeader,
|
||||||
|
pub stdout: String,
|
||||||
|
pub stderr: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Incremental text returned by `get_progress`. Carries only the new
|
||||||
|
/// bytes since the caller's last cursor positions so the SSE stream
|
||||||
|
/// handler can send deltas without re-transmitting the full log.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct BuildLogProgress {
|
||||||
|
/// New stdout bytes beyond `stdout_cursor`.
|
||||||
|
pub stdout_append: String,
|
||||||
|
/// New stderr bytes beyond `stderr_cursor`.
|
||||||
|
pub stderr_append: String,
|
||||||
|
/// `Some(unix_ts)` once the build is finished.
|
||||||
|
pub finished_at: Option<i64>,
|
||||||
|
/// Terminal status string (`"ok"` / `"fail"`) once finished.
|
||||||
|
pub status: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Channel capacity for per-build append notifications. 64 slots is
|
||||||
|
/// plenty — the consumer reads fast relative to line-append rate and
|
||||||
|
/// any lag means "read now, you have new content" rather than a lost
|
||||||
|
/// data line.
|
||||||
|
const NOTIFY_CAP: usize = 64;
|
||||||
|
|
||||||
|
/// Sqlite-backed build-log store. `Arc<BuildLogs>`-friendly: all
|
||||||
|
/// methods take `&self`, internal `Mutex<Connection>` serializes
|
||||||
|
/// access.
|
||||||
|
pub struct BuildLogs {
|
||||||
|
conn: Mutex<Connection>,
|
||||||
|
/// Broadcast channel that fires with the `id` of the row that just
|
||||||
|
/// had a line appended or was finished. The SSE stream handler
|
||||||
|
/// subscribes once per open panel and drives delta reads from this.
|
||||||
|
/// `send()` is non-async and silently drops frames when there are
|
||||||
|
/// no subscribers — safe to call from sync append/finish paths.
|
||||||
|
notify_tx: broadcast::Sender<i64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BuildLogs {
|
||||||
|
pub fn open(db_dir: &Path) -> Result<Self> {
|
||||||
|
let path = db_dir.join("build_logs.sqlite");
|
||||||
|
let conn = crate::db::open(&path, "build_logs")?;
|
||||||
|
conn.execute_batch(SCHEMA)
|
||||||
|
.context("apply build_logs schema")?;
|
||||||
|
let (notify_tx, _) = broadcast::channel(NOTIFY_CAP);
|
||||||
|
Ok(Self {
|
||||||
|
conn: Mutex::new(conn),
|
||||||
|
notify_tx,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Subscribe to per-build append/finish notifications. Each emitted
|
||||||
|
/// value is the `id` of the row that changed. The SSE stream handler
|
||||||
|
/// calls this once and filters for its target id.
|
||||||
|
pub fn subscribe_notifications(&self) -> broadcast::Receiver<i64> {
|
||||||
|
self.notify_tx.subscribe()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Open a row for a new build attempt. Returns the assigned id
|
||||||
|
/// — the caller threads it through `append_stdout` / `append_stderr`
|
||||||
|
/// while the child runs and into `finish` once it exits.
|
||||||
|
pub fn start(&self, agent: &str, kind: &str, cmdline: &str) -> Result<i64> {
|
||||||
|
let now = now_unix();
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO build_logs (agent, kind, cmdline, started_at) VALUES (?1, ?2, ?3, ?4)",
|
||||||
|
params![agent, kind, cmdline, now],
|
||||||
|
)
|
||||||
|
.context("insert build_logs row")?;
|
||||||
|
Ok(conn.last_insert_rowid())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Append a single stdout line. Best-effort: errors are logged
|
||||||
|
/// but never returned to the caller, so a transient sqlite blip
|
||||||
|
/// never tears down a rebuild's stdout pump.
|
||||||
|
pub fn append_stdout(&self, id: i64, line: &str) {
|
||||||
|
self.append(id, "stdout", line);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Append a single stderr line. Same best-effort contract as
|
||||||
|
/// `append_stdout`.
|
||||||
|
pub fn append_stderr(&self, id: i64, line: &str) {
|
||||||
|
self.append(id, "stderr", line);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn append(&self, id: i64, column: &'static str, line: &str) {
|
||||||
|
// `column` is hard-coded by the caller (`stdout` / `stderr`)
|
||||||
|
// — never user-supplied — so the string-format here is safe
|
||||||
|
// and lets us reuse one helper for both streams.
|
||||||
|
let sql = format!("UPDATE build_logs SET {column} = {column} || ?1 || x'0a' WHERE id = ?2");
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
if let Err(e) = conn.execute(&sql, params![line, id]) {
|
||||||
|
tracing::warn!(
|
||||||
|
build_log_id = id,
|
||||||
|
column = column,
|
||||||
|
error = ?e,
|
||||||
|
"build_logs: append failed (dropping line)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
drop(conn);
|
||||||
|
// Notify SSE stream subscribers — non-blocking, no-op when no
|
||||||
|
// subscribers are watching (e.g. no panel is open). Lagged
|
||||||
|
// receivers (channel full) automatically drop frames; the SSE
|
||||||
|
// handler re-reads the full delta on the next notification it
|
||||||
|
// does receive, so no content is lost, only an intermediate
|
||||||
|
// wake-up is coalesced.
|
||||||
|
let _ = self.notify_tx.send(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Finalize a build attempt. Sets `finished_at` to now and
|
||||||
|
/// `status` to the terminal state. Best-effort.
|
||||||
|
pub fn finish(&self, id: i64, status: BuildStatus) {
|
||||||
|
let now = now_unix();
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
if let Err(e) = conn.execute(
|
||||||
|
"UPDATE build_logs SET finished_at = ?1, status = ?2 WHERE id = ?3",
|
||||||
|
params![now, status.as_str(), id],
|
||||||
|
) {
|
||||||
|
tracing::warn!(
|
||||||
|
build_log_id = id,
|
||||||
|
error = ?e,
|
||||||
|
"build_logs: finish failed"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
drop(conn);
|
||||||
|
// Final notification so the SSE stream handler sees the
|
||||||
|
// finished_at and status, closes the connection cleanly.
|
||||||
|
let _ = self.notify_tx.send(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return incremental log content beyond the given byte cursors.
|
||||||
|
/// Used by the SSE stream handler to compute deltas between polls.
|
||||||
|
///
|
||||||
|
/// `stdout_cursor` / `stderr_cursor` are byte offsets into the
|
||||||
|
/// stored `stdout` / `stderr` columns from the previous read.
|
||||||
|
/// Slicing is safe because cursors are always derived from prior
|
||||||
|
/// `String::len()` values (valid UTF-8 boundaries).
|
||||||
|
///
|
||||||
|
/// Returns `None` when the row no longer exists (vacuum reap during
|
||||||
|
/// a long-open panel).
|
||||||
|
pub fn get_progress(
|
||||||
|
&self,
|
||||||
|
id: i64,
|
||||||
|
stdout_cursor: usize,
|
||||||
|
stderr_cursor: usize,
|
||||||
|
) -> Result<Option<BuildLogProgress>> {
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
let mut stmt = conn.prepare(
|
||||||
|
"SELECT stdout, stderr, finished_at, status \
|
||||||
|
FROM build_logs WHERE id = ?1",
|
||||||
|
)?;
|
||||||
|
let row = stmt
|
||||||
|
.query_row(params![id], |r| {
|
||||||
|
Ok((
|
||||||
|
r.get::<_, String>(0)?,
|
||||||
|
r.get::<_, String>(1)?,
|
||||||
|
r.get::<_, Option<i64>>(2)?,
|
||||||
|
r.get::<_, Option<String>>(3)?,
|
||||||
|
))
|
||||||
|
})
|
||||||
|
.optional()?;
|
||||||
|
match row {
|
||||||
|
None => Ok(None),
|
||||||
|
Some((stdout, stderr, finished_at, status)) => {
|
||||||
|
let stdout_append = stdout.get(stdout_cursor..).unwrap_or("").to_string();
|
||||||
|
let stderr_append = stderr.get(stderr_cursor..).unwrap_or("").to_string();
|
||||||
|
Ok(Some(BuildLogProgress {
|
||||||
|
stdout_append,
|
||||||
|
stderr_append,
|
||||||
|
finished_at,
|
||||||
|
status,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return the most recent `limit` rows for `agent`, newest first.
|
||||||
|
/// Headers only (no stdout/stderr blobs) — keeps `/api/state`
|
||||||
|
/// payloads light. Limit is hard-clamped to 50 to bound worst-case
|
||||||
|
/// payload regardless of caller input.
|
||||||
|
pub fn list_recent_for_agent(&self, agent: &str, limit: usize) -> Result<Vec<BuildLogHeader>> {
|
||||||
|
let limit = limit.min(50);
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
let mut stmt = conn.prepare(
|
||||||
|
"SELECT id, agent, kind, cmdline, started_at, finished_at, status
|
||||||
|
FROM build_logs
|
||||||
|
WHERE agent = ?1
|
||||||
|
ORDER BY started_at DESC
|
||||||
|
LIMIT ?2",
|
||||||
|
)?;
|
||||||
|
let rows = stmt.query_map(
|
||||||
|
params![agent, i64::try_from(limit).unwrap_or(50)],
|
||||||
|
row_to_header,
|
||||||
|
)?;
|
||||||
|
let mut out = Vec::new();
|
||||||
|
for r in rows {
|
||||||
|
out.push(r?);
|
||||||
|
}
|
||||||
|
Ok(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return the most recent `limit` rows across all agents, newest first.
|
||||||
|
/// Same header-only shape as `list_recent_for_agent`. Limit clamped to 100.
|
||||||
|
pub fn list_recent_all(&self, limit: usize) -> Result<Vec<BuildLogHeader>> {
|
||||||
|
let limit = limit.min(100);
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
let mut stmt = conn.prepare(
|
||||||
|
"SELECT id, agent, kind, cmdline, started_at, finished_at, status
|
||||||
|
FROM build_logs
|
||||||
|
ORDER BY started_at DESC
|
||||||
|
LIMIT ?1",
|
||||||
|
)?;
|
||||||
|
let rows = stmt.query_map(params![i64::try_from(limit).unwrap_or(100)], row_to_header)?;
|
||||||
|
let mut out = Vec::new();
|
||||||
|
for r in rows {
|
||||||
|
out.push(r?);
|
||||||
|
}
|
||||||
|
Ok(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fetch a single full row (with stdout/stderr text) by id.
|
||||||
|
/// Returns `None` when the id doesn't exist (vacuum sweep already
|
||||||
|
/// reaped it, or the operator passed a stale id from a refresh
|
||||||
|
/// race).
|
||||||
|
pub fn get_full(&self, id: i64) -> Result<Option<BuildLogFull>> {
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
let mut stmt = conn.prepare(
|
||||||
|
"SELECT id, agent, kind, cmdline, started_at, finished_at, status, stdout, stderr
|
||||||
|
FROM build_logs
|
||||||
|
WHERE id = ?1",
|
||||||
|
)?;
|
||||||
|
let row = stmt
|
||||||
|
.query_row(params![id], |r| {
|
||||||
|
let started_at: i64 = r.get(4)?;
|
||||||
|
let finished_at: Option<i64> = r.get(5)?;
|
||||||
|
let runtime_secs = compute_runtime_secs(started_at, finished_at);
|
||||||
|
Ok(BuildLogFull {
|
||||||
|
header: BuildLogHeader {
|
||||||
|
id: r.get(0)?,
|
||||||
|
agent: r.get(1)?,
|
||||||
|
kind: r.get(2)?,
|
||||||
|
cmdline: r.get(3)?,
|
||||||
|
started_at,
|
||||||
|
finished_at,
|
||||||
|
status: r.get(6)?,
|
||||||
|
runtime_secs,
|
||||||
|
},
|
||||||
|
stdout: r.get(7)?,
|
||||||
|
stderr: r.get(8)?,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.optional()?;
|
||||||
|
Ok(row)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Drop rows past their retention window. Returns the number of
|
||||||
|
/// rows deleted. Called from this module's hourly `spawn` loop.
|
||||||
|
///
|
||||||
|
/// Rule:
|
||||||
|
/// - `status = 'fail'` rows kept for `KEEP_FAIL_SECS` past their
|
||||||
|
/// `finished_at` (failures are what operators dig into).
|
||||||
|
/// - `status = 'ok'` rows kept for `KEEP_OK_SECS` past their
|
||||||
|
/// `finished_at` (successes are mostly noise after a day).
|
||||||
|
/// - In-flight rows (`finished_at IS NULL`) are never touched —
|
||||||
|
/// a long-running build shouldn't disappear from its own log
|
||||||
|
/// viewer mid-stream.
|
||||||
|
pub fn vacuum(&self) -> Result<u64> {
|
||||||
|
let now = now_unix();
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
let fail_cutoff = now - KEEP_FAIL_SECS;
|
||||||
|
let ok_cutoff = now - KEEP_OK_SECS;
|
||||||
|
let removed = conn.execute(
|
||||||
|
"DELETE FROM build_logs
|
||||||
|
WHERE finished_at IS NOT NULL
|
||||||
|
AND (
|
||||||
|
(status = 'fail' AND finished_at < ?1)
|
||||||
|
OR (status = 'ok' AND finished_at < ?2)
|
||||||
|
)",
|
||||||
|
params![fail_cutoff, ok_cutoff],
|
||||||
|
)?;
|
||||||
|
Ok(u64::try_from(removed).unwrap_or(0))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Spawn the hourly retention sweep. A host-side sweep (`build_logs.sqlite`
|
||||||
|
/// is hive-c0re-owned, so no privsep ownership issue). Runs once at startup
|
||||||
|
/// before its first sleep so a long-uptime instance doesn't accumulate a
|
||||||
|
/// backlog the first hour after restart.
|
||||||
|
pub fn spawn_vacuum(coord: &Arc<crate::coordinator::Coordinator>) {
|
||||||
|
use std::time::Duration;
|
||||||
|
let logs = coord.build_logs.clone();
|
||||||
|
let mut shutdown = coord.shutdown_rx();
|
||||||
|
let interval = Duration::from_hours(1);
|
||||||
|
tokio::spawn(async move {
|
||||||
|
loop {
|
||||||
|
match logs.vacuum() {
|
||||||
|
Ok(0) => {}
|
||||||
|
Ok(n) => tracing::info!(removed = n, "build_logs vacuum"),
|
||||||
|
Err(e) => tracing::warn!(error = ?e, "build_logs vacuum failed"),
|
||||||
|
}
|
||||||
|
tokio::select! {
|
||||||
|
() = tokio::time::sleep(interval) => {}
|
||||||
|
_ = shutdown.changed() => {
|
||||||
|
tracing::info!("build_logs vacuum: shutdown signal received");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
fn compute_runtime_secs(started_at: i64, finished_at: Option<i64>) -> Option<i64> {
|
||||||
|
finished_at.map(|f| f - started_at)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn row_to_header(r: &rusqlite::Row) -> rusqlite::Result<BuildLogHeader> {
|
||||||
|
let started_at: i64 = r.get(4)?;
|
||||||
|
let finished_at: Option<i64> = r.get(5)?;
|
||||||
|
let runtime_secs = compute_runtime_secs(started_at, finished_at);
|
||||||
|
Ok(BuildLogHeader {
|
||||||
|
id: r.get(0)?,
|
||||||
|
agent: r.get(1)?,
|
||||||
|
kind: r.get(2)?,
|
||||||
|
cmdline: r.get(3)?,
|
||||||
|
started_at,
|
||||||
|
finished_at,
|
||||||
|
status: r.get(6)?,
|
||||||
|
runtime_secs,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn tmpdb() -> (tempfile::TempDir, BuildLogs) {
|
||||||
|
let dir = tempfile::tempdir().expect("tempdir");
|
||||||
|
let db = BuildLogs::open(dir.path()).expect("open");
|
||||||
|
(dir, db)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn start_appends_finish_flow() {
|
||||||
|
let (_d, db) = tmpdb();
|
||||||
|
let id = db
|
||||||
|
.start("alice", "prebuild", "nix build foo")
|
||||||
|
.expect("start");
|
||||||
|
db.append_stdout(id, "building '/nix/store/abc.drv'");
|
||||||
|
db.append_stderr(id, "error: line 12");
|
||||||
|
db.append_stderr(id, " at /nix/store/.../module.nix:5");
|
||||||
|
db.finish(id, BuildStatus::Fail);
|
||||||
|
|
||||||
|
let full = db.get_full(id).expect("get").expect("Some");
|
||||||
|
assert_eq!(full.header.agent, "alice");
|
||||||
|
assert_eq!(full.header.kind, "prebuild");
|
||||||
|
assert_eq!(full.header.status.as_deref(), Some("fail"));
|
||||||
|
assert!(full.header.finished_at.is_some());
|
||||||
|
assert!(full.stdout.contains("/nix/store/abc.drv"));
|
||||||
|
assert!(full.stderr.contains("error: line 12"));
|
||||||
|
assert!(full.stderr.contains("module.nix:5"));
|
||||||
|
// Lines are terminator-delimited so each contributes a trailing
|
||||||
|
// newline — the viewer joins on the existing newlines rather
|
||||||
|
// than re-inserting them.
|
||||||
|
assert!(full.stderr.ends_with('\n'));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn list_recent_orders_newest_first_and_clamps() {
|
||||||
|
let (_d, db) = tmpdb();
|
||||||
|
// Three attempts, two for alice + one for bob. Without sleeping
|
||||||
|
// sqlite's `INTEGER` started_at ties at 1-sec resolution, so we
|
||||||
|
// assert id-ordering (autoincrement) is the tiebreaker — list
|
||||||
|
// sorts by started_at DESC but the ORDER BY still produces the
|
||||||
|
// last-inserted row first when timestamps match.
|
||||||
|
let id_a1 = db.start("alice", "run", "cmd one").expect("start");
|
||||||
|
let _id_b = db.start("bob", "run", "cmd two").expect("start");
|
||||||
|
let id_a2 = db.start("alice", "run", "cmd three").expect("start");
|
||||||
|
db.finish(id_a1, BuildStatus::Ok);
|
||||||
|
|
||||||
|
let alice_rows = db.list_recent_for_agent("alice", 10).expect("list");
|
||||||
|
assert_eq!(alice_rows.len(), 2);
|
||||||
|
// Without distinct started_at values both rows share `now`,
|
||||||
|
// but list_recent already orders by `started_at DESC` then
|
||||||
|
// sqlite's natural insertion-order tiebreak. We rely only on
|
||||||
|
// both IDs being present + correct count + agent isolation.
|
||||||
|
let ids: std::collections::HashSet<i64> = alice_rows.iter().map(|h| h.id).collect();
|
||||||
|
assert!(ids.contains(&id_a1));
|
||||||
|
assert!(ids.contains(&id_a2));
|
||||||
|
|
||||||
|
let bob_rows = db.list_recent_for_agent("bob", 10).expect("list");
|
||||||
|
assert_eq!(bob_rows.len(), 1);
|
||||||
|
|
||||||
|
// Limit clamp at 50.
|
||||||
|
let huge = db.list_recent_for_agent("alice", 999_999).expect("list");
|
||||||
|
assert!(huge.len() <= 50);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn get_full_returns_none_for_missing_id() {
|
||||||
|
let (_d, db) = tmpdb();
|
||||||
|
let missing = db.get_full(999_999).expect("get");
|
||||||
|
assert!(missing.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn vacuum_drops_old_finished_only_per_status() {
|
||||||
|
let (_d, db) = tmpdb();
|
||||||
|
let id_fresh_fail = db.start("alice", "run", "fresh fail").expect("start");
|
||||||
|
let id_old_fail = db.start("alice", "run", "old fail").expect("start");
|
||||||
|
let id_old_ok = db.start("alice", "run", "old ok").expect("start");
|
||||||
|
let id_running = db.start("alice", "run", "still running").expect("start");
|
||||||
|
db.finish(id_fresh_fail, BuildStatus::Fail);
|
||||||
|
db.finish(id_old_fail, BuildStatus::Fail);
|
||||||
|
db.finish(id_old_ok, BuildStatus::Ok);
|
||||||
|
// Backdate two rows past their retention windows. fresh_fail
|
||||||
|
// stays within KEEP_FAIL_SECS so it survives; old_fail goes
|
||||||
|
// beyond; old_ok goes past KEEP_OK_SECS but inside
|
||||||
|
// KEEP_FAIL_SECS — proves the per-status rule.
|
||||||
|
let now = now_unix();
|
||||||
|
{
|
||||||
|
let conn = db.conn.lock().unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE build_logs SET finished_at = ?1 WHERE id = ?2",
|
||||||
|
params![now - KEEP_FAIL_SECS - 60, id_old_fail],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE build_logs SET finished_at = ?1 WHERE id = ?2",
|
||||||
|
params![now - KEEP_OK_SECS - 60, id_old_ok],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let removed = db.vacuum().expect("vacuum");
|
||||||
|
assert_eq!(removed, 2, "old_fail + old_ok should be vacuumed");
|
||||||
|
assert!(db.get_full(id_fresh_fail).unwrap().is_some());
|
||||||
|
assert!(db.get_full(id_old_fail).unwrap().is_none());
|
||||||
|
assert!(db.get_full(id_old_ok).unwrap().is_none());
|
||||||
|
// Running row must survive vacuum regardless of retention
|
||||||
|
// windows — finished_at IS NULL gates it out.
|
||||||
|
assert!(db.get_full(id_running).unwrap().is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn append_after_finish_still_appends() {
|
||||||
|
// Defensive: if a child's stdout pump fires one last line
|
||||||
|
// between the wait-syscall returning and `finish` running, the
|
||||||
|
// append should land on the row (status already set, but the
|
||||||
|
// log stays consistent with what happened).
|
||||||
|
let (_d, db) = tmpdb();
|
||||||
|
let id = db.start("alice", "run", "cmd").expect("start");
|
||||||
|
db.finish(id, BuildStatus::Ok);
|
||||||
|
db.append_stdout(id, "post-finish trailing line");
|
||||||
|
let full = db.get_full(id).expect("get").expect("Some");
|
||||||
|
assert!(full.stdout.contains("post-finish trailing line"));
|
||||||
|
}
|
||||||
|
}
|
||||||
58
hive-c0re/src/stores/db.rs
Normal file
58
hive-c0re/src/stores/db.rs
Normal file
|
|
@ -0,0 +1,58 @@
|
||||||
|
//! Shared sqlite connection setup for hive-c0re's host-side stores.
|
||||||
|
//!
|
||||||
|
//! Several modules keep their own tables — and their own
|
||||||
|
//! `Mutex<Connection>` — in the coordinator DB
|
||||||
|
//! (`db/broker.sqlite`: broker, approvals, operator questions,
|
||||||
|
//! scheduled prompts, agent power) or in a sibling file under the same
|
||||||
|
//! `db/` dir (`build_logs.sqlite`, `audit_log.sqlite`). The open dance
|
||||||
|
//! is identical everywhere: ensure the parent dir exists, open the
|
||||||
|
//! connection, set a busy timeout so concurrent same-process writers
|
||||||
|
//! wait each other out instead of surfacing `SQLITE_BUSY`. This helper
|
||||||
|
//! owns that dance; schema creation + column migrations stay with each
|
||||||
|
//! store (they're per-table concerns).
|
||||||
|
|
||||||
|
use std::path::Path;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use anyhow::{Context, Result};
|
||||||
|
use rusqlite::Connection;
|
||||||
|
|
||||||
|
/// How long a write waits on another connection's lock before erroring.
|
||||||
|
/// Generous relative to the stores' tiny transactions — a timeout here
|
||||||
|
/// means something is genuinely wedged, not ordinary contention.
|
||||||
|
const BUSY_TIMEOUT: Duration = Duration::from_secs(5);
|
||||||
|
|
||||||
|
/// Apply additive, idempotent migrations: run each statement and
|
||||||
|
/// ignore `duplicate column name` errors (sqlite has no
|
||||||
|
/// `ADD COLUMN IF NOT EXISTS`; try-and-ignore is the portable path —
|
||||||
|
/// same pattern as hive-ag3nt's `turn_stats`). Any other error
|
||||||
|
/// propagates. Fits plain `ALTER TABLE ADD COLUMN` (new columns must
|
||||||
|
/// carry a default or tolerate NULL) and `IF NOT EXISTS` index DDL;
|
||||||
|
/// migrations with creation-conditional backfills (broker's
|
||||||
|
/// `acked_at`) stay bespoke in their store.
|
||||||
|
pub fn apply_migrations(conn: &Connection, subsystem: &str, statements: &[&str]) -> Result<()> {
|
||||||
|
for stmt in statements {
|
||||||
|
if let Err(e) = conn.execute_batch(stmt) {
|
||||||
|
if e.to_string().contains("duplicate column name") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
return Err(e).with_context(|| format!("{subsystem} migration failed: {stmt}"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Open a connection to the sqlite file at `path`, creating the parent
|
||||||
|
/// directory if needed. `subsystem` labels error contexts (`"broker"`,
|
||||||
|
/// `"approvals"`, …).
|
||||||
|
pub fn open(path: &Path, subsystem: &str) -> Result<Connection> {
|
||||||
|
if let Some(parent) = path.parent() {
|
||||||
|
std::fs::create_dir_all(parent)
|
||||||
|
.with_context(|| format!("create {subsystem} db parent {}", parent.display()))?;
|
||||||
|
}
|
||||||
|
let conn = Connection::open(path)
|
||||||
|
.with_context(|| format!("open {subsystem} db {}", path.display()))?;
|
||||||
|
conn.busy_timeout(BUSY_TIMEOUT)
|
||||||
|
.with_context(|| format!("set {subsystem} busy_timeout"))?;
|
||||||
|
Ok(conn)
|
||||||
|
}
|
||||||
14
hive-c0re/src/stores/mod.rs
Normal file
14
hive-c0re/src/stores/mod.rs
Normal file
|
|
@ -0,0 +1,14 @@
|
||||||
|
//! Sqlite-backed host-side stores (broker, approval / question /
|
||||||
|
//! schedule queues, build logs, audit trail, power intent) plus the
|
||||||
|
//! shared connection open/migration helper (`db`). Each submodule is
|
||||||
|
//! re-exported at the crate root, so `crate::broker::…` etc. keep
|
||||||
|
//! working unchanged.
|
||||||
|
|
||||||
|
pub mod approvals;
|
||||||
|
pub mod audit_log;
|
||||||
|
pub mod broker;
|
||||||
|
pub mod build_logs;
|
||||||
|
pub mod db;
|
||||||
|
pub mod operator_questions;
|
||||||
|
pub mod power;
|
||||||
|
pub mod scheduled_prompts;
|
||||||
273
hive-c0re/src/stores/operator_questions.rs
Normal file
273
hive-c0re/src/stores/operator_questions.rs
Normal file
|
|
@ -0,0 +1,273 @@
|
||||||
|
//! Question queue. Agents submit via `Ask`; the answer comes from
|
||||||
|
//! either the operator (via the dashboard, for `target IS NULL`) or
|
||||||
|
//! a peer agent (via `Answer`, for agent-to-agent questions).
|
||||||
|
//!
|
||||||
|
//! Despite the file name (kept for git history sanity), this table
|
||||||
|
//! now stores *all* asynchronous questions in the hive — both the
|
||||||
|
//! operator-targeted ones and the peer-to-peer ones. `target IS
|
||||||
|
//! NULL` is the operator path (back-compat with rows written before
|
||||||
|
//! the column existed); `target = '<agent-name>'` is the
|
||||||
|
//! agent-to-agent path.
|
||||||
|
|
||||||
|
use std::path::Path;
|
||||||
|
use std::sync::Mutex;
|
||||||
|
|
||||||
|
use anyhow::{Context, Result, bail};
|
||||||
|
|
||||||
|
use chrono::{DateTime, Utc};
|
||||||
|
use hive_sh4re::wire_time::now_unix;
|
||||||
|
use rusqlite::{Connection, OptionalExtension, params};
|
||||||
|
use serde::Serialize;
|
||||||
|
|
||||||
|
const SCHEMA: &str = r"
|
||||||
|
CREATE TABLE IF NOT EXISTS operator_questions (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
asker TEXT NOT NULL,
|
||||||
|
question TEXT NOT NULL,
|
||||||
|
options_json TEXT NOT NULL,
|
||||||
|
asked_at INTEGER NOT NULL,
|
||||||
|
answered_at INTEGER,
|
||||||
|
answer TEXT
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_operator_questions_pending
|
||||||
|
ON operator_questions (id) WHERE answered_at IS NULL;
|
||||||
|
";
|
||||||
|
|
||||||
|
/// Additive column migrations for pre-existing databases, applied via
|
||||||
|
/// `db::apply_migrations` (try-and-ignore-duplicate-column).
|
||||||
|
const MIGRATIONS: &[&str] = &[
|
||||||
|
"ALTER TABLE operator_questions ADD COLUMN multi INTEGER NOT NULL DEFAULT 0",
|
||||||
|
"ALTER TABLE operator_questions ADD COLUMN deadline_at INTEGER",
|
||||||
|
// `target` = recipient of the question. NULL = operator
|
||||||
|
// (back-compat default for rows written before agent-to-agent
|
||||||
|
// questions existed); a non-null agent name = peer-to-peer
|
||||||
|
// question. Dashboard's `pending()` filters on `target IS NULL`
|
||||||
|
// so peer questions never leak into the operator's queue.
|
||||||
|
"ALTER TABLE operator_questions ADD COLUMN target TEXT",
|
||||||
|
];
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
pub struct OpQuestion {
|
||||||
|
pub id: i64,
|
||||||
|
pub asker: String,
|
||||||
|
pub question: String,
|
||||||
|
pub options: Vec<String>,
|
||||||
|
pub multi: bool,
|
||||||
|
pub asked_at: DateTime<Utc>,
|
||||||
|
/// Deadline after which a watchdog auto-resolves the question with
|
||||||
|
/// answer `[expired]`. `None` = no expiry. Surfaced on the
|
||||||
|
/// dashboard as a remaining-time chip.
|
||||||
|
pub deadline_at: Option<DateTime<Utc>>,
|
||||||
|
pub answered_at: Option<DateTime<Utc>>,
|
||||||
|
pub answer: Option<String>,
|
||||||
|
/// Recipient of the question. `None` = the operator (dashboard
|
||||||
|
/// path); `Some(<agent>)` = a peer agent asked via
|
||||||
|
/// `Ask { to: Some(<agent>), ... }`. Agent-to-agent questions
|
||||||
|
/// never appear in `pending()` so the operator's queue stays clean.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub target: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct OperatorQuestions {
|
||||||
|
conn: Mutex<Connection>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl OperatorQuestions {
|
||||||
|
pub fn open(path: &Path) -> Result<Self> {
|
||||||
|
let conn = crate::db::open(path, "operator_questions")?;
|
||||||
|
conn.execute_batch(SCHEMA)
|
||||||
|
.context("apply operator_questions schema")?;
|
||||||
|
crate::db::apply_migrations(&conn, "operator_questions", MIGRATIONS)?;
|
||||||
|
Ok(Self {
|
||||||
|
conn: Mutex::new(conn),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn submit(
|
||||||
|
&self,
|
||||||
|
asker: &str,
|
||||||
|
question: &str,
|
||||||
|
options: &[String],
|
||||||
|
multi: bool,
|
||||||
|
deadline_at: Option<i64>,
|
||||||
|
target: Option<&str>,
|
||||||
|
) -> Result<i64> {
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
let options_json = serde_json::to_string(options).unwrap_or_else(|_| "[]".into());
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO operator_questions
|
||||||
|
(asker, question, options_json, multi, deadline_at, target, asked_at)
|
||||||
|
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
|
||||||
|
params![
|
||||||
|
asker,
|
||||||
|
question,
|
||||||
|
options_json,
|
||||||
|
i64::from(multi),
|
||||||
|
deadline_at,
|
||||||
|
target,
|
||||||
|
now_unix(),
|
||||||
|
],
|
||||||
|
)?;
|
||||||
|
Ok(conn.last_insert_rowid())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Mark a pending question answered. `answerer` is who's actually
|
||||||
|
/// answering: `"operator"` for the dashboard path, or an agent's
|
||||||
|
/// own name when responding via `Answer`. Authorisation:
|
||||||
|
///
|
||||||
|
/// - Operator-targeted questions (`target IS NULL`) can only be
|
||||||
|
/// answered by `"operator"`. (Agents must not be able to spoof
|
||||||
|
/// answers to operator questions — the dashboard is the
|
||||||
|
/// privileged path.)
|
||||||
|
/// - Agent-targeted questions can only be answered by the
|
||||||
|
/// declared target agent, OR by `"operator"` (operator override
|
||||||
|
/// for stuck threads — useful when an agent is offline/down
|
||||||
|
/// and someone has to close the loop).
|
||||||
|
///
|
||||||
|
/// Returns `(question, asker, target)` so the caller can fire the
|
||||||
|
/// `QuestionAnswered` event with the right answerer label and route
|
||||||
|
/// it back to the original asker.
|
||||||
|
pub fn answer(
|
||||||
|
&self,
|
||||||
|
id: i64,
|
||||||
|
answer: &str,
|
||||||
|
answerer: &str,
|
||||||
|
) -> Result<(String, String, Option<String>)> {
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
let row: Option<(String, String, Option<String>, Option<i64>)> = conn
|
||||||
|
.query_row(
|
||||||
|
"SELECT question, asker, target, answered_at FROM operator_questions WHERE id = ?1",
|
||||||
|
params![id],
|
||||||
|
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)),
|
||||||
|
)
|
||||||
|
.optional()?;
|
||||||
|
let Some((question, asker, target, answered_at)) = row else {
|
||||||
|
bail!("question {id} not found");
|
||||||
|
};
|
||||||
|
if answered_at.is_some() {
|
||||||
|
bail!("question {id} already answered");
|
||||||
|
}
|
||||||
|
// Authorisation check: must match the target, or be the operator
|
||||||
|
// (operator-targeted questions are operator-only; the operator
|
||||||
|
// can additionally override agent-to-agent questions to close
|
||||||
|
// stuck threads).
|
||||||
|
let authorised = match target.as_deref() {
|
||||||
|
None => answerer == hive_sh4re::OPERATOR_RECIPIENT,
|
||||||
|
Some(t) => answerer == t || answerer == hive_sh4re::OPERATOR_RECIPIENT,
|
||||||
|
};
|
||||||
|
if !authorised {
|
||||||
|
bail!(
|
||||||
|
"question {id} not addressed to '{answerer}' (target = {:?})",
|
||||||
|
target.as_deref().unwrap_or(hive_sh4re::OPERATOR_RECIPIENT)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE operator_questions SET answer = ?1, answered_at = ?2 WHERE id = ?3",
|
||||||
|
params![answer, now_unix(), id],
|
||||||
|
)?;
|
||||||
|
Ok((question, asker, target))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Cancel a pending question on behalf of `canceller`. Returns
|
||||||
|
/// `(question, asker, target)` so the caller can fire the usual
|
||||||
|
/// `QuestionAnswered` event to the asker with a `[cancelled by
|
||||||
|
/// <canceller>]` sentinel.
|
||||||
|
///
|
||||||
|
/// Auth: the canceller must be one of:
|
||||||
|
/// - the original asker (an agent withdrawing their own ask),
|
||||||
|
/// - the operator (already covered by the existing `answer` path
|
||||||
|
/// but allowed here too for symmetry / dashboard cancel),
|
||||||
|
/// - a `privileged` caller (one that arrived on the manager socket —
|
||||||
|
/// privileged hive-wide cleanup; derived from the socket, not a
|
||||||
|
/// name match).
|
||||||
|
///
|
||||||
|
/// Not the target — that's covered by `answer` (responding with
|
||||||
|
/// an actual reply, sentinel or otherwise).
|
||||||
|
pub fn cancel(
|
||||||
|
&self,
|
||||||
|
id: i64,
|
||||||
|
canceller: &str,
|
||||||
|
privileged: bool,
|
||||||
|
) -> Result<(String, String, Option<String>)> {
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
let row: Option<(String, String, Option<String>, Option<i64>)> = conn
|
||||||
|
.query_row(
|
||||||
|
"SELECT question, asker, target, answered_at FROM operator_questions WHERE id = ?1",
|
||||||
|
params![id],
|
||||||
|
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)),
|
||||||
|
)
|
||||||
|
.optional()?;
|
||||||
|
let Some((question, asker, target, answered_at)) = row else {
|
||||||
|
bail!("question {id} not found");
|
||||||
|
};
|
||||||
|
if answered_at.is_some() {
|
||||||
|
bail!("question {id} already answered/cancelled");
|
||||||
|
}
|
||||||
|
let authorised =
|
||||||
|
privileged || canceller == asker || canceller == hive_sh4re::OPERATOR_RECIPIENT;
|
||||||
|
if !authorised {
|
||||||
|
bail!("question {id}: '{canceller}' not allowed to cancel (asker = '{asker}')");
|
||||||
|
}
|
||||||
|
let sentinel = format!("[cancelled by {canceller}]");
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE operator_questions SET answer = ?1, answered_at = ?2 WHERE id = ?3",
|
||||||
|
params![sentinel, now_unix(), id],
|
||||||
|
)?;
|
||||||
|
Ok((question, asker, target))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every pending question, operator-targeted or peer-to-peer.
|
||||||
|
/// Drives the dashboard's questions pane now that peer threads
|
||||||
|
/// are surfaced for visibility + operator override-answer.
|
||||||
|
pub fn pending_all(&self) -> Result<Vec<OpQuestion>> {
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
let mut stmt = conn.prepare(
|
||||||
|
"SELECT id, asker, question, options_json, multi, asked_at, answered_at, answer, deadline_at, target
|
||||||
|
FROM operator_questions
|
||||||
|
WHERE answered_at IS NULL
|
||||||
|
ORDER BY id ASC",
|
||||||
|
)?;
|
||||||
|
let rows = stmt.query_map([], row_to_question)?;
|
||||||
|
rows.collect::<rusqlite::Result<Vec<_>>>()
|
||||||
|
.map_err(Into::into)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Last `limit` answered questions across both target kinds,
|
||||||
|
/// newest-first. Companion to `pending_all`.
|
||||||
|
pub fn recent_answered_all(&self, limit: u64) -> Result<Vec<OpQuestion>> {
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
let mut stmt = conn.prepare(
|
||||||
|
"SELECT id, asker, question, options_json, multi, asked_at, answered_at, answer, deadline_at, target
|
||||||
|
FROM operator_questions
|
||||||
|
WHERE answered_at IS NOT NULL
|
||||||
|
ORDER BY answered_at DESC
|
||||||
|
LIMIT ?1",
|
||||||
|
)?;
|
||||||
|
let limit_i = i64::try_from(limit).unwrap_or(i64::MAX);
|
||||||
|
let rows = stmt.query_map(params![limit_i], row_to_question)?;
|
||||||
|
rows.collect::<rusqlite::Result<Vec<_>>>()
|
||||||
|
.map_err(Into::into)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn row_to_question(row: &rusqlite::Row<'_>) -> rusqlite::Result<OpQuestion> {
|
||||||
|
let options_json: String = row.get(3)?;
|
||||||
|
let options: Vec<String> = serde_json::from_str(&options_json).unwrap_or_default();
|
||||||
|
let multi: i64 = row.get(4)?;
|
||||||
|
Ok(OpQuestion {
|
||||||
|
id: row.get(0)?,
|
||||||
|
asker: row.get(1)?,
|
||||||
|
question: row.get(2)?,
|
||||||
|
options,
|
||||||
|
multi: multi != 0,
|
||||||
|
asked_at: hive_sh4re::wire_time::from_secs(row.get(5)?),
|
||||||
|
answered_at: row
|
||||||
|
.get::<_, Option<i64>>(6)?
|
||||||
|
.map(hive_sh4re::wire_time::from_secs),
|
||||||
|
answer: row.get(7)?,
|
||||||
|
deadline_at: row
|
||||||
|
.get::<_, Option<i64>>(8)?
|
||||||
|
.map(hive_sh4re::wire_time::from_secs),
|
||||||
|
target: row.get(9)?,
|
||||||
|
})
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue