vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

refactor(hive-c0re): group src-root files into submodules

Browse source 
stores/ (sqlite-backed host stores + db helper), stats/, agent_config/,
workers/ — pure git-mv moves; crate-root re-exports keep every
crate::<module> path compiling. flake_check stays at root (synchronous
approval-flow validation, not a background worker)
This commit is contained in:
müde 2026-07-06 22:38:47 +02:00
commit 0e4b5a1120
29 changed files with 68 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

515
hive-c0re/src/stores/approvals.rs Normal file
View file

@ -0,0 +1,515 @@
//! Approval queue. Manager submits via `RequestApplyCommit`; the user
//! approves/denies via the host admin CLI; on approval the host runs the
//! corresponding action (Phase 5a: `lifecycle::rebuild(agent)`).
use std::path::Path;
use std::sync::Mutex;
use anyhow::{Context, Result, bail};
use hive_sh4re::wire_time::now_unix;
use hive_sh4re::{Approval, ApprovalKind, ApprovalStatus};
use rusqlite::{Connection, OptionalExtension, params};
const SCHEMA: &str = r"
CREATE TABLE IF NOT EXISTS approvals (
id INTEGER PRIMARY KEY AUTOINCREMENT,
agent TEXT NOT NULL,
commit_ref TEXT NOT NULL,
requested_at INTEGER NOT NULL,
status TEXT NOT NULL,
resolved_at INTEGER,
note TEXT
);
CREATE INDEX IF NOT EXISTS idx_approvals_pending
ON approvals (id) WHERE status = 'pending';
";
/// Additive column migrations for pre-existing databases, applied via
/// `db::apply_migrations` (try-and-ignore-duplicate-column).
const MIGRATIONS: &[&str] = &[
// `kind` (pre-Phase-8 dbs): legacy rows default to `apply_commit`,
// which matches their actual semantics.
"ALTER TABLE approvals ADD COLUMN kind TEXT NOT NULL DEFAULT 'apply_commit'",
// `description`: manager-supplied note shown on the dashboard
// approval card at submission time (distinct from `note`, set on
// denial/failure).
"ALTER TABLE approvals ADD COLUMN description TEXT",
// `fetched_sha`: the canonical sha hive-c0re vouched for at
// `request_apply_commit` time. Distinct from `commit_ref`
// (manager-supplied, may not even resolve by approve time).
"ALTER TABLE approvals ADD COLUMN fetched_sha TEXT",
// `submitter`: the agent that submitted the approval (the
// authenticated socket caller); approval-scoped helper events
// route to it. Legacy rows are NULL → callers fall back to the
// root agent.
"ALTER TABLE approvals ADD COLUMN submitter TEXT",
];
pub struct Approvals {
conn: Mutex<Connection>,
}
impl Approvals {
pub fn open(path: &Path) -> Result<Self> {
let conn = crate::db::open(path, "approvals")?;
conn.execute_batch(SCHEMA)
.context("apply approvals schema")?;
crate::db::apply_migrations(&conn, "approvals", MIGRATIONS)?;
Ok(Self {
conn: Mutex::new(conn),
})
}
pub fn submit_kind(
&self,
agent: &str,
kind: ApprovalKind,
commit_ref: &str,
description: Option<&str>,
submitter: &str,
) -> Result<i64> {
let conn = self.conn.lock().unwrap();
conn.execute(
"INSERT INTO approvals
(agent, kind, commit_ref, requested_at, status, description, submitter)
VALUES (?1, ?2, ?3, ?4, 'pending', ?5, ?6)",
params![
agent,
kind.as_str(),
commit_ref,
now_unix(),
description,
submitter
],
)?;
Ok(conn.last_insert_rowid())
}
/// The agent that submitted approval `id` (the authenticated socket
/// caller at submit time). `None` for legacy rows predating the
/// `submitter` column — callers route those to the root agent.
///
/// # Errors
///
/// Returns an error if the sqlite prepare/query fails. A missing row
/// or a `NULL` submitter is not an error — both yield `Ok(None)`.
pub fn submitter_of(&self, id: i64) -> Result<Option<String>> {
let conn = self.conn.lock().unwrap();
let submitter: Option<String> = conn
.query_row(
"SELECT submitter FROM approvals WHERE id = ?1",
params![id],
|row| row.get(0),
)
.optional()?
.flatten();
Ok(submitter)
}
/// Record the canonical sha hive-c0re fetched from the proposed repo
/// into applied at submission time. Idempotent on identical values.
pub fn set_fetched_sha(&self, id: i64, sha: &str) -> Result<()> {
let conn = self.conn.lock().unwrap();
conn.execute(
"UPDATE approvals SET fetched_sha = ?1 WHERE id = ?2",
params![sha, id],
)?;
Ok(())
}
/// Last `limit` resolved approvals (approved / denied / failed),
/// newest-first. Drives the history tab on the dashboard.
pub fn recent_resolved(&self, limit: u64) -> Result<Vec<Approval>> {
let conn = self.conn.lock().unwrap();
let mut stmt = conn.prepare(
"SELECT id, agent, kind, commit_ref, requested_at, status, resolved_at, note, fetched_sha, description
FROM approvals
WHERE status IN ('approved', 'denied', 'failed', 'cancelled')
ORDER BY resolved_at DESC, id DESC
LIMIT ?1",
)?;
let rows = stmt.query_map([limit], row_to_approval)?;
Ok(collect_lenient(rows))
}
pub fn pending(&self) -> Result<Vec<Approval>> {
let conn = self.conn.lock().unwrap();
let mut stmt = conn.prepare(
"SELECT id, agent, kind, commit_ref, requested_at, status, resolved_at, note, fetched_sha, description
FROM approvals
WHERE status = 'pending'
ORDER BY id ASC",
)?;
let rows = stmt.query_map([], row_to_approval)?;
Ok(collect_lenient(rows))
}
pub fn get(&self, id: i64) -> Result<Option<Approval>> {
let conn = self.conn.lock().unwrap();
conn.query_row(
"SELECT id, agent, kind, commit_ref, requested_at, status, resolved_at, note, fetched_sha, description
FROM approvals WHERE id = ?1",
params![id],
row_to_approval,
)
.optional()
.map_err(Into::into)
}
/// Mark pending -> approved (or fail if not pending). Returns the (now-updated)
/// approval so the caller can run the action and pass the agent name.
pub fn mark_approved(&self, id: i64) -> Result<Approval> {
let conn = self.conn.lock().unwrap();
let row: Option<ApprovalLookup> = conn
.query_row(
ApprovalLookup::SELECT,
params![id],
ApprovalLookup::from_row,
)
.optional()?;
let Some(row) = row else {
bail!("approval {id} not found");
};
if row.status != "pending" {
bail!("approval {id} is {}, not pending", row.status);
}
let resolved_at = now_unix();
conn.execute(
"UPDATE approvals SET status = 'approved', resolved_at = ?1 WHERE id = ?2",
params![resolved_at, id],
)?;
Ok(Approval {
id,
agent: row.agent,
kind: kind_from_str(&row.kind)?,
commit_ref: row.commit_ref,
requested_at: hive_sh4re::wire_time::from_secs(row.requested_at),
status: ApprovalStatus::Approved,
resolved_at: Some(hive_sh4re::wire_time::from_secs(resolved_at)),
note: None,
fetched_sha: row.fetched_sha,
description: row.description,
})
}
pub fn mark_denied(&self, id: i64, note: Option<&str>) -> Result<()> {
let conn = self.conn.lock().unwrap();
let affected = conn.execute(
"UPDATE approvals SET status = 'denied', resolved_at = ?1, note = ?2
WHERE id = ?3 AND status = 'pending'",
params![now_unix(), note, id],
)?;
if affected == 0 {
bail!("approval {id} not pending");
}
Ok(())
}
pub fn mark_failed(&self, id: i64, note: &str) -> Result<()> {
let conn = self.conn.lock().unwrap();
conn.execute(
"UPDATE approvals SET status = 'failed', resolved_at = ?1, note = ?2 WHERE id = ?3",
params![now_unix(), note, id],
)?;
Ok(())
}
/// Withdraw a pending approval. Returns the now-updated
/// row so the caller can emit `ApprovalResolved` with the right
/// kind / agent / sha. Errors if the approval isn't pending — once
/// it's approved/denied/failed/cancelled, the resolution is final.
pub fn mark_cancelled(&self, id: i64, canceller: &str) -> Result<Approval> {
let mut conn = self.conn.lock().unwrap();
let tx = conn.transaction()?;
let row: Option<ApprovalLookup> = tx
.query_row(
ApprovalLookup::SELECT,
params![id],
ApprovalLookup::from_row,
)
.optional()?;
let Some(row) = row else {
bail!("approval {id} not found");
};
if row.status != "pending" {
bail!("approval {id} is {}, not pending", row.status);
}
let resolved_at = now_unix();
let note = format!("cancelled by {canceller}");
tx.execute(
"UPDATE approvals SET status = 'cancelled', resolved_at = ?1, note = ?2 WHERE id = ?3",
params![resolved_at, note, id],
)?;
tx.commit()?;
Ok(Approval {
id,
agent: row.agent,
kind: kind_from_str(&row.kind)?,
commit_ref: row.commit_ref,
requested_at: hive_sh4re::wire_time::from_secs(row.requested_at),
status: ApprovalStatus::Cancelled,
resolved_at: Some(hive_sh4re::wire_time::from_secs(resolved_at)),
note: Some(note),
fetched_sha: row.fetched_sha,
description: row.description,
})
}
/// Mark every pending approval for `agent` as failed (returns rows affected).
/// Used by `destroy` to clear the queue of an agent that no longer exists.
pub fn fail_pending_for_agent(&self, agent: &str, note: &str) -> Result<usize> {
let conn = self.conn.lock().unwrap();
let n = conn.execute(
"UPDATE approvals SET status = 'failed', resolved_at = ?1, note = ?2
WHERE agent = ?3 AND status = 'pending'",
params![now_unix(), note, agent],
)?;
Ok(n)
}
}
/// Columns needed to rebuild an [`Approval`] after a status transition,
/// shared by `mark_approved` / `mark_cancelled`. Replaces a 7-field
/// tuple that tripped `clippy::type_complexity` and was duplicated
/// across both callers (one suppressed the lint, the other aliased the
/// tuple) — one named projection + mapper now backs both.
struct ApprovalLookup {
agent: String,
kind: String,
commit_ref: String,
requested_at: i64,
status: String,
fetched_sha: Option<String>,
description: Option<String>,
}
impl ApprovalLookup {
/// The single-row lookup by id (`?1`). Column order matches
/// [`ApprovalLookup::from_row`].
const SELECT: &str = "SELECT agent, kind, commit_ref, requested_at, status, fetched_sha, \
description FROM approvals WHERE id = ?1";
fn from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<Self> {
Ok(Self {
agent: row.get(0)?,
kind: row.get(1)?,
commit_ref: row.get(2)?,
requested_at: row.get(3)?,
status: row.get(4)?,
fetched_sha: row.get(5)?,
description: row.get(6)?,
})
}
}
/// Collect approval rows, dropping (and logging) any that fail to
/// deserialize. A single malformed / unknown-kind row must never blank
/// the whole list: `collect::<Result<Vec>>()` is all-or-nothing, so one
/// bad row used to make `pending()` / `recent_resolved()` error out
/// wholesale — the dashboard then rendered an empty approvals queue.
fn collect_lenient(rows: impl Iterator<Item = rusqlite::Result<Approval>>) -> Vec<Approval> {
rows.filter_map(|r| match r {
Ok(a) => Some(a),
Err(e) => {
tracing::warn!(error = ?e, "skipping unparseable approval row");
None
}
})
.collect()
}
fn row_to_approval(row: &rusqlite::Row<'_>) -> rusqlite::Result<Approval> {
// Column order: id, agent, kind, commit_ref, requested_at, status, resolved_at, note, fetched_sha, description.
let kind: String = row.get(2)?;
let kind = match kind.as_str() {
"apply_commit" => ApprovalKind::ApplyCommit,
"spawn" => ApprovalKind::Spawn,
"init_config" => ApprovalKind::InitConfig,
"update_meta_inputs" => ApprovalKind::UpdateMetaInputs,
"schedule_prompt" => ApprovalKind::SchedulePrompt,
"merge_config_pr" => ApprovalKind::MergeConfigPr,
other => {
return Err(rusqlite::Error::FromSqlConversionFailure(
2,
rusqlite::types::Type::Text,
format!("unknown approval kind '{other}'").into(),
));
}
};
let status: String = row.get(5)?;
let status = match status.as_str() {
"pending" => ApprovalStatus::Pending,
"approved" => ApprovalStatus::Approved,
"denied" => ApprovalStatus::Denied,
"failed" => ApprovalStatus::Failed,
"cancelled" => ApprovalStatus::Cancelled,
other => {
return Err(rusqlite::Error::FromSqlConversionFailure(
5,
rusqlite::types::Type::Text,
format!("unknown approval status '{other}'").into(),
));
}
};
Ok(Approval {
id: row.get(0)?,
agent: row.get(1)?,
kind,
commit_ref: row.get(3)?,
requested_at: hive_sh4re::wire_time::from_secs(row.get(4)?),
status,
resolved_at: row
.get::<_, Option<i64>>(6)?
.map(hive_sh4re::wire_time::from_secs),
note: row.get(7)?,
fetched_sha: row.get(8)?,
description: row.get(9)?,
})
}
fn kind_from_str(s: &str) -> Result<ApprovalKind> {
Ok(match s {
"apply_commit" => ApprovalKind::ApplyCommit,
"spawn" => ApprovalKind::Spawn,
"init_config" => ApprovalKind::InitConfig,
"update_meta_inputs" => ApprovalKind::UpdateMetaInputs,
"schedule_prompt" => ApprovalKind::SchedulePrompt,
"merge_config_pr" => ApprovalKind::MergeConfigPr,
other => bail!("unknown approval kind '{other}'"),
})
}
#[cfg(test)]
mod tests {
use super::*;
use hive_sh4re::ApprovalKind;
fn open_temp() -> (tempfile::TempDir, std::path::PathBuf, Approvals) {
let dir = tempfile::tempdir().expect("tempdir");
let path = dir.path().join("approvals.sqlite");
let db = Approvals::open(&path).expect("open approvals db");
(dir, path, db)
}
#[test]
fn init_config_approval_round_trips() {
// Regression test: an `init_config` row used to fail
// deserialization (row_to_approval matched only apply_commit +
// spawn), erroring out the whole `pending()` query — every
// approval then vanished from the dashboard.
let (_dir, _path, db) = open_temp();
let id = db
.submit_kind(
"bitburner",
ApprovalKind::InitConfig,
"",
Some("scaffold"),
"bitburner",
)
.expect("submit init_config");
let pending = db
.pending()
.expect("pending() must not error on an init_config row");
assert_eq!(pending.len(), 1);
assert_eq!(pending[0].id, id);
assert!(matches!(pending[0].kind, ApprovalKind::InitConfig));
}
#[test]
fn mixed_kinds_all_listed() {
let (_dir, _path, db) = open_temp();
db.submit_kind("a", ApprovalKind::ApplyCommit, "deadbeef", None, "a")
.unwrap();
db.submit_kind("b", ApprovalKind::Spawn, "", None, "b")
.unwrap();
db.submit_kind("c", ApprovalKind::InitConfig, "", None, "c")
.unwrap();
let pending = db.pending().expect("pending");
assert_eq!(pending.len(), 3, "all three kinds must be visible");
}
#[test]
fn mark_cancelled_transitions_pending_row() {
// Manager withdraws a pending approval. Row leaves pending(),
// shows up in recent_resolved() with the cancelled status + a
// "cancelled by <who>" note.
let (_dir, _path, db) = open_temp();
let id = db
.submit_kind(
"bitburner",
ApprovalKind::ApplyCommit,
"cafef00d",
Some("test"),
"bitburner",
)
.unwrap();
let row = db.mark_cancelled(id, "manager").expect("cancel");
assert_eq!(row.id, id);
assert!(matches!(row.status, ApprovalStatus::Cancelled));
assert_eq!(row.note.as_deref(), Some("cancelled by manager"));
assert!(row.resolved_at.is_some());
assert!(db.pending().unwrap().is_empty(), "row leaves pending");
let resolved = db.recent_resolved(10).unwrap();
assert_eq!(resolved.len(), 1);
assert!(matches!(resolved[0].status, ApprovalStatus::Cancelled));
}
#[test]
fn mark_cancelled_refuses_already_resolved_row() {
// Once approved/denied/failed/cancelled the resolution is
// final — re-cancelling errors instead of silently overwriting.
let (_dir, _path, db) = open_temp();
let id = db
.submit_kind("a", ApprovalKind::Spawn, "deadbeef", None, "a")
.unwrap();
db.mark_cancelled(id, "manager").expect("first cancel");
let err = db
.mark_cancelled(id, "manager")
.expect_err("second cancel must fail");
assert!(err.to_string().contains("not pending"), "{err}");
}
#[test]
fn unknown_kind_row_is_skipped_not_fatal() {
// A single malformed / future-kind row must not blank the
// whole list — collect_lenient skips it instead of failing.
let (_dir, path, db) = open_temp();
let good = db
.submit_kind("good", ApprovalKind::ApplyCommit, "cafe", None, "good")
.unwrap();
let raw = Connection::open(&path).unwrap();
raw.execute(
"INSERT INTO approvals (agent, kind, commit_ref, requested_at, status)
VALUES ('weird', 'from_the_future', '', 0, 'pending')",
[],
)
.unwrap();
let pending = db
.pending()
.expect("pending() must survive an unparseable row");
assert_eq!(pending.len(), 1);
assert_eq!(pending[0].id, good);
}
#[test]
fn submitter_round_trips_and_legacy_reads_none() {
// A submitted approval records its submitter; a legacy row
// (inserted without the column) reads back as None so callers
// fall back to the root agent.
let (_dir, path, db) = open_temp();
let id = db
.submit_kind("child", ApprovalKind::ApplyCommit, "cafe", None, "parent")
.unwrap();
assert_eq!(db.submitter_of(id).unwrap().as_deref(), Some("parent"));
let raw = Connection::open(&path).unwrap();
raw.execute(
"INSERT INTO approvals (agent, kind, commit_ref, requested_at, status)
VALUES ('old', 'apply_commit', '', 0, 'pending')",
[],
)
.unwrap();
let legacy_id = raw.last_insert_rowid();
assert_eq!(db.submitter_of(legacy_id).unwrap(), None);
}
}

332
hive-c0re/src/stores/audit_log.rs Normal file
View file

@ -0,0 +1,332 @@
//! Sqlite-backed audit trail of agent-initiated privileged actions.
//!
//! Surfaces, durably and operator-visibly, the privileged operations
//! hive-c0re performs *on behalf of an agent* — the ones that cross the
//! agent/operator trust boundary and so warrant a who/what/when record
//! beyond hive-priv's low-level journal trace. First entry: infra
//! container restarts via the `infra_admin`-gated `restart` tool (the
//! follow-up audit trail for that capability).
//!
//! Deliberately scoped to *agent-initiated* privileged actions. The bulk
//! of `PrivRequest` traffic (token writes, nspawn-flag edits) fires
//! constantly during normal lifecycle and is hive-c0re's own bookkeeping,
//! not an agent crossing the boundary — logging all of it would drown the
//! signal the operator actually wants.
//!
//! Same process-singleton handle pattern as `build_logs`: installed once
//! at `Coordinator::open`, fetched via [`global`] so the recording sites
//! (e.g. `socket_server::handle_restart_infra`) don't have to thread an
//! `Arc<AuditLog>` through every call path. Recording is best-effort: a
//! sqlite blip must never fail the underlying privileged action.
use std::path::Path;
use std::sync::{Arc, Mutex, OnceLock};
use anyhow::{Context, Result};
use chrono::{DateTime, Utc};
use hive_sh4re::wire_time::now_unix;
use rusqlite::{Connection, params};
use serde::Serialize;
/// Process-singleton handle, set once at coordinator startup. Mirrors
/// `build_logs::GLOBAL` — lets recording sites write without threading an
/// `Arc<AuditLog>` through every entry point.
static GLOBAL: OnceLock<Arc<AuditLog>> = OnceLock::new();
/// Install the process-wide `AuditLog` handle. Idempotent: a second call
/// silently keeps the first handle.
pub fn install(handle: Arc<AuditLog>) {
let _ = GLOBAL.set(handle);
}
/// Fetch the process-wide handle, or `None` if `install` hasn't run yet
/// (early startup, or unit tests). Callers must gracefully no-op on `None`.
#[must_use]
pub fn global() -> Option<Arc<AuditLog>> {
GLOBAL.get().cloned()
}
/// Retain audit rows for 90 days. Longer than build-log retention — this
/// is a security/accountability record, not debug noise; the operator may
/// want to review "who restarted what" well after the fact.
const KEEP_SECS: i64 = 90 * 24 * 3600;
const SCHEMA: &str = "
CREATE TABLE IF NOT EXISTS audit_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
ts_unix INTEGER NOT NULL,
agent TEXT NOT NULL,
action TEXT NOT NULL,
target TEXT NOT NULL,
outcome TEXT NOT NULL,
detail TEXT
);
CREATE INDEX IF NOT EXISTS idx_audit_log_ts ON audit_log (ts_unix DESC);
";
/// Outcome of a recorded privileged action. Stored as the literal string
/// in the `outcome` column.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum AuditOutcome {
/// The privileged action succeeded.
Ok,
/// The privileged action was attempted but failed (e.g. the
/// underlying systemctl call errored). Denied-by-capability attempts
/// are recorded too — see the recording site.
Err,
}
impl AuditOutcome {
fn as_str(self) -> &'static str {
match self {
Self::Ok => "ok",
Self::Err => "err",
}
}
}
/// One audit row as returned to the dashboard.
#[derive(Debug, Clone, Serialize)]
pub struct AuditEntry {
pub id: i64,
pub ts_unix: DateTime<Utc>,
/// Agent on whose behalf the action was taken.
pub agent: String,
/// What was done (e.g. `restart_infra`).
pub action: String,
/// What it acted on (e.g. `hive-ci`).
pub target: String,
/// `"ok"` | `"err"`.
pub outcome: String,
/// Optional free-text detail (e.g. the error message on failure).
pub detail: Option<String>,
}
/// Sqlite-backed audit-log store. `Arc<AuditLog>`-friendly: all methods
/// take `&self`, an internal `Mutex<Connection>` serializes access.
pub struct AuditLog {
conn: Mutex<Connection>,
}
impl AuditLog {
/// Open (creating if absent) the `audit_log.sqlite` store under
/// `db_dir` and apply the schema. `db_dir` is shared with
/// `build_logs` (the broker db's parent directory).
///
/// # Errors
/// Returns an error if the directory can't be created, the sqlite
/// file can't be opened, or applying the schema fails.
pub fn open(db_dir: &Path) -> Result<Self> {
let path = db_dir.join("audit_log.sqlite");
let conn = crate::db::open(&path, "audit_log")?;
conn.execute_batch(SCHEMA)
.context("apply audit_log schema")?;
Ok(Self {
conn: Mutex::new(conn),
})
}
/// Record one privileged action. Best-effort: a sqlite error is logged
/// but never returned, so a transient blip never fails the underlying
/// privileged action (the action already happened — losing its audit
/// row is strictly less bad than failing the action retroactively).
///
/// Returns the inserted [`AuditEntry`] (with its assigned id +
/// timestamp) on success, or `None` if the insert failed. The
/// returned row is the canonical record — callers that also push a
/// live event (e.g. the dashboard stream) emit *this* rather than
/// re-deriving the fields, so the stored row and the streamed event
/// can't drift.
#[must_use]
pub fn record(
&self,
agent: &str,
action: &str,
target: &str,
outcome: AuditOutcome,
detail: Option<&str>,
) -> Option<AuditEntry> {
let now = now_unix();
let conn = self.conn.lock().unwrap();
match conn.execute(
"INSERT INTO audit_log (ts_unix, agent, action, target, outcome, detail)
VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
params![now, agent, action, target, outcome.as_str(), detail],
) {
Ok(_) => Some(AuditEntry {
id: conn.last_insert_rowid(),
ts_unix: hive_sh4re::wire_time::from_secs(now),
agent: agent.to_owned(),
action: action.to_owned(),
target: target.to_owned(),
outcome: outcome.as_str().to_owned(),
detail: detail.map(str::to_owned),
}),
Err(e) => {
tracing::warn!(
%agent, %action, %target,
error = ?e,
"audit_log: record failed (dropping entry)"
);
None
}
}
}
/// Return the most recent `limit` rows, newest first. Limit is
/// hard-clamped to 500 to bound the worst-case payload.
///
/// # Errors
/// Returns an error if the query fails to prepare or a row fails to
/// deserialize.
pub fn list_recent(&self, limit: usize) -> Result<Vec<AuditEntry>> {
let limit = limit.min(500);
let conn = self.conn.lock().unwrap();
let mut stmt = conn.prepare(
"SELECT id, ts_unix, agent, action, target, outcome, detail
FROM audit_log
ORDER BY ts_unix DESC, id DESC
LIMIT ?1",
)?;
let rows = stmt.query_map(params![i64::try_from(limit).unwrap_or(500)], row_to_entry)?;
let mut out = Vec::new();
for r in rows {
out.push(r?);
}
Ok(out)
}
/// Total row count, regardless of the `list_recent` clamp. Lets the
/// dashboard show "latest N of TOTAL" instead of silently capping.
///
/// # Errors
/// Returns an error if the `COUNT(*)` query fails.
pub fn count_total(&self) -> Result<i64> {
let conn = self.conn.lock().unwrap();
let n: i64 = conn.query_row("SELECT COUNT(*) FROM audit_log", [], |r| r.get(0))?;
Ok(n)
}
/// Drop rows older than the retention window. Returns the number of
/// rows deleted. Called from the hourly vacuum loop.
///
/// # Errors
/// Returns an error if the `DELETE` query fails.
pub fn vacuum(&self) -> Result<u64> {
let cutoff = now_unix() - KEEP_SECS;
let conn = self.conn.lock().unwrap();
let removed = conn.execute("DELETE FROM audit_log WHERE ts_unix < ?1", params![cutoff])?;
Ok(u64::try_from(removed).unwrap_or(0))
}
}
/// Spawn the hourly retention sweep. Mirrors `build_logs::spawn_vacuum`
/// in cadence + shutdown handling.
pub fn spawn_vacuum(coord: &Arc<crate::coordinator::Coordinator>) {
use std::time::Duration;
let audit = coord.audit_log.clone();
let mut shutdown = coord.shutdown_rx();
let interval = Duration::from_hours(1);
tokio::spawn(async move {
loop {
match audit.vacuum() {
Ok(0) => {}
Ok(n) => tracing::info!(removed = n, "audit_log vacuum"),
Err(e) => tracing::warn!(error = ?e, "audit_log vacuum failed"),
}
tokio::select! {
() = tokio::time::sleep(interval) => {}
_ = shutdown.changed() => {
tracing::info!("audit_log vacuum: shutdown signal received");
break;
}
}
}
});
}
fn row_to_entry(r: &rusqlite::Row) -> rusqlite::Result<AuditEntry> {
Ok(AuditEntry {
id: r.get(0)?,
ts_unix: hive_sh4re::wire_time::from_secs(r.get(1)?),
agent: r.get(2)?,
action: r.get(3)?,
target: r.get(4)?,
outcome: r.get(5)?,
detail: r.get(6)?,
})
}
#[cfg(test)]
mod tests {
use super::*;
fn tmpdb() -> (tempfile::TempDir, AuditLog) {
let dir = tempfile::tempdir().expect("tempdir");
let db = AuditLog::open(dir.path()).expect("open");
(dir, db)
}
#[test]
fn record_and_list_newest_first() {
let (_d, db) = tmpdb();
// record() returns the canonical inserted row (id + ts assigned).
let entry = db
.record("atlas", "restart_infra", "hive-ci", AuditOutcome::Ok, None)
.expect("record returns the inserted entry");
assert!(entry.id > 0);
assert_eq!(entry.target, "hive-ci");
assert_eq!(entry.outcome, "ok");
assert!(entry.detail.is_none());
let _ = db.record(
"atlas",
"restart_infra",
"hive-gateway",
AuditOutcome::Err,
Some("systemctl failed"),
);
let rows = db.list_recent(10).expect("list");
assert_eq!(rows.len(), 2);
// Newest first: the gateway/err row was inserted last.
assert_eq!(rows[0].target, "hive-gateway");
assert_eq!(rows[0].outcome, "err");
assert_eq!(rows[0].detail.as_deref(), Some("systemctl failed"));
assert_eq!(rows[1].target, "hive-ci");
assert_eq!(rows[1].outcome, "ok");
assert!(rows[1].detail.is_none());
assert_eq!(rows[0].agent, "atlas");
assert_eq!(rows[0].action, "restart_infra");
assert_eq!(db.count_total().expect("count"), 2);
}
#[test]
fn list_clamps_to_500() {
let (_d, db) = tmpdb();
let _ = db.record("a", "x", "t", AuditOutcome::Ok, None);
let rows = db.list_recent(999_999).expect("list");
assert!(rows.len() <= 500);
}
#[test]
fn vacuum_drops_only_old_rows() {
let (_d, db) = tmpdb();
let _ = db.record("a", "restart_infra", "hive-ci", AuditOutcome::Ok, None);
// Backdate it past the retention window.
{
let conn = db.conn.lock().unwrap();
conn.execute(
"UPDATE audit_log SET ts_unix = ?1",
params![now_unix() - KEEP_SECS - 60],
)
.unwrap();
}
let _ = db.record("a", "restart_infra", "hive-forge", AuditOutcome::Ok, None);
let removed = db.vacuum().expect("vacuum");
assert_eq!(removed, 1, "only the backdated row should be reaped");
let rows = db.list_recent(10).expect("list");
assert_eq!(rows.len(), 1);
assert_eq!(rows[0].target, "hive-forge");
}
}

1578
hive-c0re/src/stores/broker.rs Normal file
View file

File diff suppressed because it is too large Load diff

565
hive-c0re/src/stores/build_logs.rs Normal file
View file

@ -0,0 +1,565 @@
//! Sqlite-backed full build-log capture — stdout + stderr per
//! `nixos-container` / `nix build` invocation, accumulated live.
//! Schema, indices, retention, and the rationale for replacing
//! the old ring buffer: `docs/persistence.md::/var/lib/hyperhive/db/build_logs.sqlite`.
use std::path::Path;
use std::sync::{Arc, Mutex, OnceLock};
use anyhow::{Context, Result};
use hive_sh4re::wire_time::now_unix;
use rusqlite::{Connection, OptionalExtension, params};
use serde::Serialize;
use tokio::sync::broadcast;
/// Process-singleton handle, set once at coordinator startup. Lets
/// the `lifecycle` module's `run` / `prebuild_toplevel` access the
/// writer without threading an `Arc<BuildLogs>` through every
/// `pub async fn` entry point in the lifecycle surface — there are
/// 10+ callsites and the handle is the same `Arc` everywhere
/// anyway. Set by `Coordinator::open`.
static GLOBAL: OnceLock<Arc<BuildLogs>> = OnceLock::new();
/// Install the process-wide `BuildLogs` handle. Idempotent: a second
/// call (e.g. test harness setup) silently keeps the first handle.
pub fn install(handle: Arc<BuildLogs>) {
let _ = GLOBAL.set(handle);
}
/// Fetch the process-wide handle, or `None` if `install` hasn't run
/// yet (e.g. early in startup before `Coordinator::open`, or in unit
/// tests that don't bother with the global). Callers must gracefully
/// no-op when this returns `None`.
#[must_use]
pub fn global() -> Option<Arc<BuildLogs>> {
GLOBAL.get().cloned()
}
/// Retain failed-build rows for 30 days — they're what the operator
/// needs to investigate when diagnosing a regression.
const KEEP_FAIL_SECS: i64 = 30 * 24 * 3600;
/// Retain successful-build rows for 24 hours — useful for diffing
/// what changed across a recent rebuild, but past a day the log is
/// noise. In-progress rows are never reaped (they have
/// `finished_at IS NULL`).
const KEEP_OK_SECS: i64 = 24 * 3600;
const SCHEMA: &str = "
CREATE TABLE IF NOT EXISTS build_logs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
agent TEXT NOT NULL,
kind TEXT NOT NULL,
cmdline TEXT NOT NULL,
started_at INTEGER NOT NULL,
finished_at INTEGER,
status TEXT,
stdout TEXT NOT NULL DEFAULT '',
stderr TEXT NOT NULL DEFAULT ''
);
CREATE INDEX IF NOT EXISTS idx_build_logs_agent_started
ON build_logs (agent, started_at DESC);
CREATE INDEX IF NOT EXISTS idx_build_logs_status_finished
ON build_logs (status, finished_at)
WHERE finished_at IS NOT NULL;
";
/// Status of a finished build attempt. Stored as the literal string in
/// the `status` column; `NULL` while the attempt is still in progress.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum BuildStatus {
/// Child exited with success.
Ok,
/// Child exited non-zero (build / eval failure).
Fail,
}
impl BuildStatus {
fn as_str(self) -> &'static str {
match self {
Self::Ok => "ok",
Self::Fail => "fail",
}
}
}
/// Header-only row returned by `list_recent_for_agent`. Carries the
/// metadata the dashboard's agent-card chip needs (status + age +
/// id-to-open) without the multi-MB stdout/stderr payload.
#[derive(Debug, Clone, Serialize)]
pub struct BuildLogHeader {
pub id: i64,
pub agent: String,
pub kind: String,
pub cmdline: String,
pub started_at: i64,
pub finished_at: Option<i64>,
pub status: Option<String>,
/// Elapsed seconds from `started_at` to `finished_at`. `None` while
/// the build is still in progress.
pub runtime_secs: Option<i64>,
}
/// Full row with stdout/stderr text inlined. Returned by `get_full`,
/// backs the side-panel viewer's payload.
#[derive(Debug, Clone, Serialize)]
pub struct BuildLogFull {
#[serde(flatten)]
pub header: BuildLogHeader,
pub stdout: String,
pub stderr: String,
}
/// Incremental text returned by `get_progress`. Carries only the new
/// bytes since the caller's last cursor positions so the SSE stream
/// handler can send deltas without re-transmitting the full log.
#[derive(Debug, Clone)]
pub struct BuildLogProgress {
/// New stdout bytes beyond `stdout_cursor`.
pub stdout_append: String,
/// New stderr bytes beyond `stderr_cursor`.
pub stderr_append: String,
/// `Some(unix_ts)` once the build is finished.
pub finished_at: Option<i64>,
/// Terminal status string (`"ok"` / `"fail"`) once finished.
pub status: Option<String>,
}
/// Channel capacity for per-build append notifications. 64 slots is
/// plenty — the consumer reads fast relative to line-append rate and
/// any lag means "read now, you have new content" rather than a lost
/// data line.
const NOTIFY_CAP: usize = 64;
/// Sqlite-backed build-log store. `Arc<BuildLogs>`-friendly: all
/// methods take `&self`, internal `Mutex<Connection>` serializes
/// access.
pub struct BuildLogs {
conn: Mutex<Connection>,
/// Broadcast channel that fires with the `id` of the row that just
/// had a line appended or was finished. The SSE stream handler
/// subscribes once per open panel and drives delta reads from this.
/// `send()` is non-async and silently drops frames when there are
/// no subscribers — safe to call from sync append/finish paths.
notify_tx: broadcast::Sender<i64>,
}
impl BuildLogs {
pub fn open(db_dir: &Path) -> Result<Self> {
let path = db_dir.join("build_logs.sqlite");
let conn = crate::db::open(&path, "build_logs")?;
conn.execute_batch(SCHEMA)
.context("apply build_logs schema")?;
let (notify_tx, _) = broadcast::channel(NOTIFY_CAP);
Ok(Self {
conn: Mutex::new(conn),
notify_tx,
})
}
/// Subscribe to per-build append/finish notifications. Each emitted
/// value is the `id` of the row that changed. The SSE stream handler
/// calls this once and filters for its target id.
pub fn subscribe_notifications(&self) -> broadcast::Receiver<i64> {
self.notify_tx.subscribe()
}
/// Open a row for a new build attempt. Returns the assigned id
/// — the caller threads it through `append_stdout` / `append_stderr`
/// while the child runs and into `finish` once it exits.
pub fn start(&self, agent: &str, kind: &str, cmdline: &str) -> Result<i64> {
let now = now_unix();
let conn = self.conn.lock().unwrap();
conn.execute(
"INSERT INTO build_logs (agent, kind, cmdline, started_at) VALUES (?1, ?2, ?3, ?4)",
params![agent, kind, cmdline, now],
)
.context("insert build_logs row")?;
Ok(conn.last_insert_rowid())
}
/// Append a single stdout line. Best-effort: errors are logged
/// but never returned to the caller, so a transient sqlite blip
/// never tears down a rebuild's stdout pump.
pub fn append_stdout(&self, id: i64, line: &str) {
self.append(id, "stdout", line);
}
/// Append a single stderr line. Same best-effort contract as
/// `append_stdout`.
pub fn append_stderr(&self, id: i64, line: &str) {
self.append(id, "stderr", line);
}
fn append(&self, id: i64, column: &'static str, line: &str) {
// `column` is hard-coded by the caller (`stdout` / `stderr`)
// — never user-supplied — so the string-format here is safe
// and lets us reuse one helper for both streams.
let sql = format!("UPDATE build_logs SET {column} = {column} || ?1 || x'0a' WHERE id = ?2");
let conn = self.conn.lock().unwrap();
if let Err(e) = conn.execute(&sql, params![line, id]) {
tracing::warn!(
build_log_id = id,
column = column,
error = ?e,
"build_logs: append failed (dropping line)"
);
}
drop(conn);
// Notify SSE stream subscribers — non-blocking, no-op when no
// subscribers are watching (e.g. no panel is open). Lagged
// receivers (channel full) automatically drop frames; the SSE
// handler re-reads the full delta on the next notification it
// does receive, so no content is lost, only an intermediate
// wake-up is coalesced.
let _ = self.notify_tx.send(id);
}
/// Finalize a build attempt. Sets `finished_at` to now and
/// `status` to the terminal state. Best-effort.
pub fn finish(&self, id: i64, status: BuildStatus) {
let now = now_unix();
let conn = self.conn.lock().unwrap();
if let Err(e) = conn.execute(
"UPDATE build_logs SET finished_at = ?1, status = ?2 WHERE id = ?3",
params![now, status.as_str(), id],
) {
tracing::warn!(
build_log_id = id,
error = ?e,
"build_logs: finish failed"
);
}
drop(conn);
// Final notification so the SSE stream handler sees the
// finished_at and status, closes the connection cleanly.
let _ = self.notify_tx.send(id);
}
/// Return incremental log content beyond the given byte cursors.
/// Used by the SSE stream handler to compute deltas between polls.
///
/// `stdout_cursor` / `stderr_cursor` are byte offsets into the
/// stored `stdout` / `stderr` columns from the previous read.
/// Slicing is safe because cursors are always derived from prior
/// `String::len()` values (valid UTF-8 boundaries).
///
/// Returns `None` when the row no longer exists (vacuum reap during
/// a long-open panel).
pub fn get_progress(
&self,
id: i64,
stdout_cursor: usize,
stderr_cursor: usize,
) -> Result<Option<BuildLogProgress>> {
let conn = self.conn.lock().unwrap();
let mut stmt = conn.prepare(
"SELECT stdout, stderr, finished_at, status \
FROM build_logs WHERE id = ?1",
)?;
let row = stmt
.query_row(params![id], |r| {
Ok((
r.get::<_, String>(0)?,
r.get::<_, String>(1)?,
r.get::<_, Option<i64>>(2)?,
r.get::<_, Option<String>>(3)?,
))
})
.optional()?;
match row {
None => Ok(None),
Some((stdout, stderr, finished_at, status)) => {
let stdout_append = stdout.get(stdout_cursor..).unwrap_or("").to_string();
let stderr_append = stderr.get(stderr_cursor..).unwrap_or("").to_string();
Ok(Some(BuildLogProgress {
stdout_append,
stderr_append,
finished_at,
status,
}))
}
}
}
/// Return the most recent `limit` rows for `agent`, newest first.
/// Headers only (no stdout/stderr blobs) — keeps `/api/state`
/// payloads light. Limit is hard-clamped to 50 to bound worst-case
/// payload regardless of caller input.
pub fn list_recent_for_agent(&self, agent: &str, limit: usize) -> Result<Vec<BuildLogHeader>> {
let limit = limit.min(50);
let conn = self.conn.lock().unwrap();
let mut stmt = conn.prepare(
"SELECT id, agent, kind, cmdline, started_at, finished_at, status
FROM build_logs
WHERE agent = ?1
ORDER BY started_at DESC
LIMIT ?2",
)?;
let rows = stmt.query_map(
params![agent, i64::try_from(limit).unwrap_or(50)],
row_to_header,
)?;
let mut out = Vec::new();
for r in rows {
out.push(r?);
}
Ok(out)
}
/// Return the most recent `limit` rows across all agents, newest first.
/// Same header-only shape as `list_recent_for_agent`. Limit clamped to 100.
pub fn list_recent_all(&self, limit: usize) -> Result<Vec<BuildLogHeader>> {
let limit = limit.min(100);
let conn = self.conn.lock().unwrap();
let mut stmt = conn.prepare(
"SELECT id, agent, kind, cmdline, started_at, finished_at, status
FROM build_logs
ORDER BY started_at DESC
LIMIT ?1",
)?;
let rows = stmt.query_map(params![i64::try_from(limit).unwrap_or(100)], row_to_header)?;
let mut out = Vec::new();
for r in rows {
out.push(r?);
}
Ok(out)
}
/// Fetch a single full row (with stdout/stderr text) by id.
/// Returns `None` when the id doesn't exist (vacuum sweep already
/// reaped it, or the operator passed a stale id from a refresh
/// race).
pub fn get_full(&self, id: i64) -> Result<Option<BuildLogFull>> {
let conn = self.conn.lock().unwrap();
let mut stmt = conn.prepare(
"SELECT id, agent, kind, cmdline, started_at, finished_at, status, stdout, stderr
FROM build_logs
WHERE id = ?1",
)?;
let row = stmt
.query_row(params![id], |r| {
let started_at: i64 = r.get(4)?;
let finished_at: Option<i64> = r.get(5)?;
let runtime_secs = compute_runtime_secs(started_at, finished_at);
Ok(BuildLogFull {
header: BuildLogHeader {
id: r.get(0)?,
agent: r.get(1)?,
kind: r.get(2)?,
cmdline: r.get(3)?,
started_at,
finished_at,
status: r.get(6)?,
runtime_secs,
},
stdout: r.get(7)?,
stderr: r.get(8)?,
})
})
.optional()?;
Ok(row)
}
/// Drop rows past their retention window. Returns the number of
/// rows deleted. Called from this module's hourly `spawn` loop.
///
/// Rule:
/// - `status = 'fail'` rows kept for `KEEP_FAIL_SECS` past their
/// `finished_at` (failures are what operators dig into).
/// - `status = 'ok'` rows kept for `KEEP_OK_SECS` past their
/// `finished_at` (successes are mostly noise after a day).
/// - In-flight rows (`finished_at IS NULL`) are never touched —
/// a long-running build shouldn't disappear from its own log
/// viewer mid-stream.
pub fn vacuum(&self) -> Result<u64> {
let now = now_unix();
let conn = self.conn.lock().unwrap();
let fail_cutoff = now - KEEP_FAIL_SECS;
let ok_cutoff = now - KEEP_OK_SECS;
let removed = conn.execute(
"DELETE FROM build_logs
WHERE finished_at IS NOT NULL
AND (
(status = 'fail' AND finished_at < ?1)
OR (status = 'ok' AND finished_at < ?2)
)",
params![fail_cutoff, ok_cutoff],
)?;
Ok(u64::try_from(removed).unwrap_or(0))
}
}
/// Spawn the hourly retention sweep. A host-side sweep (`build_logs.sqlite`
/// is hive-c0re-owned, so no privsep ownership issue). Runs once at startup
/// before its first sleep so a long-uptime instance doesn't accumulate a
/// backlog the first hour after restart.
pub fn spawn_vacuum(coord: &Arc<crate::coordinator::Coordinator>) {
use std::time::Duration;
let logs = coord.build_logs.clone();
let mut shutdown = coord.shutdown_rx();
let interval = Duration::from_hours(1);
tokio::spawn(async move {
loop {
match logs.vacuum() {
Ok(0) => {}
Ok(n) => tracing::info!(removed = n, "build_logs vacuum"),
Err(e) => tracing::warn!(error = ?e, "build_logs vacuum failed"),
}
tokio::select! {
() = tokio::time::sleep(interval) => {}
_ = shutdown.changed() => {
tracing::info!("build_logs vacuum: shutdown signal received");
break;
}
}
}
});
}
fn compute_runtime_secs(started_at: i64, finished_at: Option<i64>) -> Option<i64> {
finished_at.map(|f| f - started_at)
}
fn row_to_header(r: &rusqlite::Row) -> rusqlite::Result<BuildLogHeader> {
let started_at: i64 = r.get(4)?;
let finished_at: Option<i64> = r.get(5)?;
let runtime_secs = compute_runtime_secs(started_at, finished_at);
Ok(BuildLogHeader {
id: r.get(0)?,
agent: r.get(1)?,
kind: r.get(2)?,
cmdline: r.get(3)?,
started_at,
finished_at,
status: r.get(6)?,
runtime_secs,
})
}
#[cfg(test)]
mod tests {
use super::*;
fn tmpdb() -> (tempfile::TempDir, BuildLogs) {
let dir = tempfile::tempdir().expect("tempdir");
let db = BuildLogs::open(dir.path()).expect("open");
(dir, db)
}
#[test]
fn start_appends_finish_flow() {
let (_d, db) = tmpdb();
let id = db
.start("alice", "prebuild", "nix build foo")
.expect("start");
db.append_stdout(id, "building '/nix/store/abc.drv'");
db.append_stderr(id, "error: line 12");
db.append_stderr(id, " at /nix/store/.../module.nix:5");
db.finish(id, BuildStatus::Fail);
let full = db.get_full(id).expect("get").expect("Some");
assert_eq!(full.header.agent, "alice");
assert_eq!(full.header.kind, "prebuild");
assert_eq!(full.header.status.as_deref(), Some("fail"));
assert!(full.header.finished_at.is_some());
assert!(full.stdout.contains("/nix/store/abc.drv"));
assert!(full.stderr.contains("error: line 12"));
assert!(full.stderr.contains("module.nix:5"));
// Lines are terminator-delimited so each contributes a trailing
// newline — the viewer joins on the existing newlines rather
// than re-inserting them.
assert!(full.stderr.ends_with('\n'));
}
#[test]
fn list_recent_orders_newest_first_and_clamps() {
let (_d, db) = tmpdb();
// Three attempts, two for alice + one for bob. Without sleeping
// sqlite's `INTEGER` started_at ties at 1-sec resolution, so we
// assert id-ordering (autoincrement) is the tiebreaker — list
// sorts by started_at DESC but the ORDER BY still produces the
// last-inserted row first when timestamps match.
let id_a1 = db.start("alice", "run", "cmd one").expect("start");
let _id_b = db.start("bob", "run", "cmd two").expect("start");
let id_a2 = db.start("alice", "run", "cmd three").expect("start");
db.finish(id_a1, BuildStatus::Ok);
let alice_rows = db.list_recent_for_agent("alice", 10).expect("list");
assert_eq!(alice_rows.len(), 2);
// Without distinct started_at values both rows share `now`,
// but list_recent already orders by `started_at DESC` then
// sqlite's natural insertion-order tiebreak. We rely only on
// both IDs being present + correct count + agent isolation.
let ids: std::collections::HashSet<i64> = alice_rows.iter().map(|h| h.id).collect();
assert!(ids.contains(&id_a1));
assert!(ids.contains(&id_a2));
let bob_rows = db.list_recent_for_agent("bob", 10).expect("list");
assert_eq!(bob_rows.len(), 1);
// Limit clamp at 50.
let huge = db.list_recent_for_agent("alice", 999_999).expect("list");
assert!(huge.len() <= 50);
}
#[test]
fn get_full_returns_none_for_missing_id() {
let (_d, db) = tmpdb();
let missing = db.get_full(999_999).expect("get");
assert!(missing.is_none());
}
#[test]
fn vacuum_drops_old_finished_only_per_status() {
let (_d, db) = tmpdb();
let id_fresh_fail = db.start("alice", "run", "fresh fail").expect("start");
let id_old_fail = db.start("alice", "run", "old fail").expect("start");
let id_old_ok = db.start("alice", "run", "old ok").expect("start");
let id_running = db.start("alice", "run", "still running").expect("start");
db.finish(id_fresh_fail, BuildStatus::Fail);
db.finish(id_old_fail, BuildStatus::Fail);
db.finish(id_old_ok, BuildStatus::Ok);
// Backdate two rows past their retention windows. fresh_fail
// stays within KEEP_FAIL_SECS so it survives; old_fail goes
// beyond; old_ok goes past KEEP_OK_SECS but inside
// KEEP_FAIL_SECS — proves the per-status rule.
let now = now_unix();
{
let conn = db.conn.lock().unwrap();
conn.execute(
"UPDATE build_logs SET finished_at = ?1 WHERE id = ?2",
params![now - KEEP_FAIL_SECS - 60, id_old_fail],
)
.unwrap();
conn.execute(
"UPDATE build_logs SET finished_at = ?1 WHERE id = ?2",
params![now - KEEP_OK_SECS - 60, id_old_ok],
)
.unwrap();
}
let removed = db.vacuum().expect("vacuum");
assert_eq!(removed, 2, "old_fail + old_ok should be vacuumed");
assert!(db.get_full(id_fresh_fail).unwrap().is_some());
assert!(db.get_full(id_old_fail).unwrap().is_none());
assert!(db.get_full(id_old_ok).unwrap().is_none());
// Running row must survive vacuum regardless of retention
// windows — finished_at IS NULL gates it out.
assert!(db.get_full(id_running).unwrap().is_some());
}
#[test]
fn append_after_finish_still_appends() {
// Defensive: if a child's stdout pump fires one last line
// between the wait-syscall returning and `finish` running, the
// append should land on the row (status already set, but the
// log stays consistent with what happened).
let (_d, db) = tmpdb();
let id = db.start("alice", "run", "cmd").expect("start");
db.finish(id, BuildStatus::Ok);
db.append_stdout(id, "post-finish trailing line");
let full = db.get_full(id).expect("get").expect("Some");
assert!(full.stdout.contains("post-finish trailing line"));
}
}

58
hive-c0re/src/stores/db.rs Normal file
View file

@ -0,0 +1,58 @@
//! Shared sqlite connection setup for hive-c0re's host-side stores.
//!
//! Several modules keep their own tables — and their own
//! `Mutex<Connection>` — in the coordinator DB
//! (`db/broker.sqlite`: broker, approvals, operator questions,
//! scheduled prompts, agent power) or in a sibling file under the same
//! `db/` dir (`build_logs.sqlite`, `audit_log.sqlite`). The open dance
//! is identical everywhere: ensure the parent dir exists, open the
//! connection, set a busy timeout so concurrent same-process writers
//! wait each other out instead of surfacing `SQLITE_BUSY`. This helper
//! owns that dance; schema creation + column migrations stay with each
//! store (they're per-table concerns).
use std::path::Path;
use std::time::Duration;
use anyhow::{Context, Result};
use rusqlite::Connection;
/// How long a write waits on another connection's lock before erroring.
/// Generous relative to the stores' tiny transactions — a timeout here
/// means something is genuinely wedged, not ordinary contention.
const BUSY_TIMEOUT: Duration = Duration::from_secs(5);
/// Apply additive, idempotent migrations: run each statement and
/// ignore `duplicate column name` errors (sqlite has no
/// `ADD COLUMN IF NOT EXISTS`; try-and-ignore is the portable path —
/// same pattern as hive-ag3nt's `turn_stats`). Any other error
/// propagates. Fits plain `ALTER TABLE ADD COLUMN` (new columns must
/// carry a default or tolerate NULL) and `IF NOT EXISTS` index DDL;
/// migrations with creation-conditional backfills (broker's
/// `acked_at`) stay bespoke in their store.
pub fn apply_migrations(conn: &Connection, subsystem: &str, statements: &[&str]) -> Result<()> {
for stmt in statements {
if let Err(e) = conn.execute_batch(stmt) {
if e.to_string().contains("duplicate column name") {
continue;
}
return Err(e).with_context(|| format!("{subsystem} migration failed: {stmt}"));
}
}
Ok(())
}
/// Open a connection to the sqlite file at `path`, creating the parent
/// directory if needed. `subsystem` labels error contexts (`"broker"`,
/// `"approvals"`, …).
pub fn open(path: &Path, subsystem: &str) -> Result<Connection> {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)
.with_context(|| format!("create {subsystem} db parent {}", parent.display()))?;
}
let conn = Connection::open(path)
.with_context(|| format!("open {subsystem} db {}", path.display()))?;
conn.busy_timeout(BUSY_TIMEOUT)
.with_context(|| format!("set {subsystem} busy_timeout"))?;
Ok(conn)
}

14
hive-c0re/src/stores/mod.rs Normal file
View file

@ -0,0 +1,14 @@
//! Sqlite-backed host-side stores (broker, approval / question /
//! schedule queues, build logs, audit trail, power intent) plus the
//! shared connection open/migration helper (`db`). Each submodule is
//! re-exported at the crate root, so `crate::broker::…` etc. keep
//! working unchanged.
pub mod approvals;
pub mod audit_log;
pub mod broker;
pub mod build_logs;
pub mod db;
pub mod operator_questions;
pub mod power;
pub mod scheduled_prompts;

273
hive-c0re/src/stores/operator_questions.rs Normal file
View file

@ -0,0 +1,273 @@
//! Question queue. Agents submit via `Ask`; the answer comes from
//! either the operator (via the dashboard, for `target IS NULL`) or
//! a peer agent (via `Answer`, for agent-to-agent questions).
//!
//! Despite the file name (kept for git history sanity), this table
//! now stores *all* asynchronous questions in the hive — both the
//! operator-targeted ones and the peer-to-peer ones. `target IS
//! NULL` is the operator path (back-compat with rows written before
//! the column existed); `target = '<agent-name>'` is the
//! agent-to-agent path.
use std::path::Path;
use std::sync::Mutex;
use anyhow::{Context, Result, bail};
use chrono::{DateTime, Utc};
use hive_sh4re::wire_time::now_unix;
use rusqlite::{Connection, OptionalExtension, params};
use serde::Serialize;
const SCHEMA: &str = r"
CREATE TABLE IF NOT EXISTS operator_questions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
asker TEXT NOT NULL,
question TEXT NOT NULL,
options_json TEXT NOT NULL,
asked_at INTEGER NOT NULL,
answered_at INTEGER,
answer TEXT
);
CREATE INDEX IF NOT EXISTS idx_operator_questions_pending
ON operator_questions (id) WHERE answered_at IS NULL;
";
/// Additive column migrations for pre-existing databases, applied via
/// `db::apply_migrations` (try-and-ignore-duplicate-column).
const MIGRATIONS: &[&str] = &[
"ALTER TABLE operator_questions ADD COLUMN multi INTEGER NOT NULL DEFAULT 0",
"ALTER TABLE operator_questions ADD COLUMN deadline_at INTEGER",
// `target` = recipient of the question. NULL = operator
// (back-compat default for rows written before agent-to-agent
// questions existed); a non-null agent name = peer-to-peer
// question. Dashboard's `pending()` filters on `target IS NULL`
// so peer questions never leak into the operator's queue.
"ALTER TABLE operator_questions ADD COLUMN target TEXT",
];
#[derive(Debug, Clone, Serialize)]
pub struct OpQuestion {
pub id: i64,
pub asker: String,
pub question: String,
pub options: Vec<String>,
pub multi: bool,
pub asked_at: DateTime<Utc>,
/// Deadline after which a watchdog auto-resolves the question with
/// answer `[expired]`. `None` = no expiry. Surfaced on the
/// dashboard as a remaining-time chip.
pub deadline_at: Option<DateTime<Utc>>,
pub answered_at: Option<DateTime<Utc>>,
pub answer: Option<String>,
/// Recipient of the question. `None` = the operator (dashboard
/// path); `Some(<agent>)` = a peer agent asked via
/// `Ask { to: Some(<agent>), ... }`. Agent-to-agent questions
/// never appear in `pending()` so the operator's queue stays clean.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub target: Option<String>,
}
pub struct OperatorQuestions {
conn: Mutex<Connection>,
}
impl OperatorQuestions {
pub fn open(path: &Path) -> Result<Self> {
let conn = crate::db::open(path, "operator_questions")?;
conn.execute_batch(SCHEMA)
.context("apply operator_questions schema")?;
crate::db::apply_migrations(&conn, "operator_questions", MIGRATIONS)?;
Ok(Self {
conn: Mutex::new(conn),
})
}
pub fn submit(
&self,
asker: &str,
question: &str,
options: &[String],
multi: bool,
deadline_at: Option<i64>,
target: Option<&str>,
) -> Result<i64> {
let conn = self.conn.lock().unwrap();
let options_json = serde_json::to_string(options).unwrap_or_else(|_| "[]".into());
conn.execute(
"INSERT INTO operator_questions
(asker, question, options_json, multi, deadline_at, target, asked_at)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
params![
asker,
question,
options_json,
i64::from(multi),
deadline_at,
target,
now_unix(),
],
)?;
Ok(conn.last_insert_rowid())
}
/// Mark a pending question answered. `answerer` is who's actually
/// answering: `"operator"` for the dashboard path, or an agent's
/// own name when responding via `Answer`. Authorisation:
///
/// - Operator-targeted questions (`target IS NULL`) can only be
/// answered by `"operator"`. (Agents must not be able to spoof
/// answers to operator questions — the dashboard is the
/// privileged path.)
/// - Agent-targeted questions can only be answered by the
/// declared target agent, OR by `"operator"` (operator override
/// for stuck threads — useful when an agent is offline/down
/// and someone has to close the loop).
///
/// Returns `(question, asker, target)` so the caller can fire the
/// `QuestionAnswered` event with the right answerer label and route
/// it back to the original asker.
pub fn answer(
&self,
id: i64,
answer: &str,
answerer: &str,
) -> Result<(String, String, Option<String>)> {
let conn = self.conn.lock().unwrap();
let row: Option<(String, String, Option<String>, Option<i64>)> = conn
.query_row(
"SELECT question, asker, target, answered_at FROM operator_questions WHERE id = ?1",
params![id],
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)),
)
.optional()?;
let Some((question, asker, target, answered_at)) = row else {
bail!("question {id} not found");
};
if answered_at.is_some() {
bail!("question {id} already answered");
}
// Authorisation check: must match the target, or be the operator
// (operator-targeted questions are operator-only; the operator
// can additionally override agent-to-agent questions to close
// stuck threads).
let authorised = match target.as_deref() {
None => answerer == hive_sh4re::OPERATOR_RECIPIENT,
Some(t) => answerer == t || answerer == hive_sh4re::OPERATOR_RECIPIENT,
};
if !authorised {
bail!(
"question {id} not addressed to '{answerer}' (target = {:?})",
target.as_deref().unwrap_or(hive_sh4re::OPERATOR_RECIPIENT)
);
}
conn.execute(
"UPDATE operator_questions SET answer = ?1, answered_at = ?2 WHERE id = ?3",
params![answer, now_unix(), id],
)?;
Ok((question, asker, target))
}
/// Cancel a pending question on behalf of `canceller`. Returns
/// `(question, asker, target)` so the caller can fire the usual
/// `QuestionAnswered` event to the asker with a `[cancelled by
/// <canceller>]` sentinel.
///
/// Auth: the canceller must be one of:
/// - the original asker (an agent withdrawing their own ask),
/// - the operator (already covered by the existing `answer` path
/// but allowed here too for symmetry / dashboard cancel),
/// - a `privileged` caller (one that arrived on the manager socket —
/// privileged hive-wide cleanup; derived from the socket, not a
/// name match).
///
/// Not the target — that's covered by `answer` (responding with
/// an actual reply, sentinel or otherwise).
pub fn cancel(
&self,
id: i64,
canceller: &str,
privileged: bool,
) -> Result<(String, String, Option<String>)> {
let conn = self.conn.lock().unwrap();
let row: Option<(String, String, Option<String>, Option<i64>)> = conn
.query_row(
"SELECT question, asker, target, answered_at FROM operator_questions WHERE id = ?1",
params![id],
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)),
)
.optional()?;
let Some((question, asker, target, answered_at)) = row else {
bail!("question {id} not found");
};
if answered_at.is_some() {
bail!("question {id} already answered/cancelled");
}
let authorised =
privileged || canceller == asker || canceller == hive_sh4re::OPERATOR_RECIPIENT;
if !authorised {
bail!("question {id}: '{canceller}' not allowed to cancel (asker = '{asker}')");
}
let sentinel = format!("[cancelled by {canceller}]");
conn.execute(
"UPDATE operator_questions SET answer = ?1, answered_at = ?2 WHERE id = ?3",
params![sentinel, now_unix(), id],
)?;
Ok((question, asker, target))
}
/// Every pending question, operator-targeted or peer-to-peer.
/// Drives the dashboard's questions pane now that peer threads
/// are surfaced for visibility + operator override-answer.
pub fn pending_all(&self) -> Result<Vec<OpQuestion>> {
let conn = self.conn.lock().unwrap();
let mut stmt = conn.prepare(
"SELECT id, asker, question, options_json, multi, asked_at, answered_at, answer, deadline_at, target
FROM operator_questions
WHERE answered_at IS NULL
ORDER BY id ASC",
)?;
let rows = stmt.query_map([], row_to_question)?;
rows.collect::<rusqlite::Result<Vec<_>>>()
.map_err(Into::into)
}
/// Last `limit` answered questions across both target kinds,
/// newest-first. Companion to `pending_all`.
pub fn recent_answered_all(&self, limit: u64) -> Result<Vec<OpQuestion>> {
let conn = self.conn.lock().unwrap();
let mut stmt = conn.prepare(
"SELECT id, asker, question, options_json, multi, asked_at, answered_at, answer, deadline_at, target
FROM operator_questions
WHERE answered_at IS NOT NULL
ORDER BY answered_at DESC
LIMIT ?1",
)?;
let limit_i = i64::try_from(limit).unwrap_or(i64::MAX);
let rows = stmt.query_map(params![limit_i], row_to_question)?;
rows.collect::<rusqlite::Result<Vec<_>>>()
.map_err(Into::into)
}
}
fn row_to_question(row: &rusqlite::Row<'_>) -> rusqlite::Result<OpQuestion> {
let options_json: String = row.get(3)?;
let options: Vec<String> = serde_json::from_str(&options_json).unwrap_or_default();
let multi: i64 = row.get(4)?;
Ok(OpQuestion {
id: row.get(0)?,
asker: row.get(1)?,
question: row.get(2)?,
options,
multi: multi != 0,
asked_at: hive_sh4re::wire_time::from_secs(row.get(5)?),
answered_at: row
.get::<_, Option<i64>>(6)?
.map(hive_sh4re::wire_time::from_secs),
answer: row.get(7)?,
deadline_at: row
.get::<_, Option<i64>>(8)?
.map(hive_sh4re::wire_time::from_secs),
target: row.get(9)?,
})
}

198
hive-c0re/src/stores/power.rs Normal file
View file

@ -0,0 +1,198 @@
//! Durable per-agent power *intent* (`wanted: Up | Offline`) — the
//! spec half of spec-vs-status desired-state reconciliation.
//! `container_view` remains the observed *status*; the job queue's
//! `Reconcile` nodes are the mechanism that converges the two.
//!
//! Stored as the `agent_power` table in the coordinator DB
//! (`/var/lib/hyperhive/db/broker.sqlite`, one tiny row per agent) —
//! same one-file-many-modules pattern as `approvals` /
//! `operator_questions` / `scheduled_prompts`, each with its own
//! connection. Intent persists across hive-c0re restarts; in-flight
//! queue work deliberately does not. Setting `wanted` is never a
//! queued node: operator/intent actions update the row synchronously
//! at request time, then submit the DAG whose terminal `Reconcile`
//! reads the fresh value — rapid toggles are last-writer-wins and the
//! reconciles converge. Power toggles never commit to the meta repo.
use std::path::Path;
use std::sync::Mutex;
use anyhow::{Context, Result};
use hive_sh4re::wire_time::now_unix;
use rusqlite::{Connection, OptionalExtension, params};
const SCHEMA: &str = "
CREATE TABLE IF NOT EXISTS agent_power (
agent TEXT PRIMARY KEY,
wanted TEXT NOT NULL,
updated_at INTEGER NOT NULL
);
";
/// Per-agent power intent.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Wanted {
Up,
Offline,
}
impl Wanted {
pub fn as_str(self) -> &'static str {
match self {
Wanted::Up => "up",
Wanted::Offline => "offline",
}
}
fn parse(s: &str) -> Option<Self> {
match s {
"up" => Some(Wanted::Up),
"offline" => Some(Wanted::Offline),
_ => None,
}
}
/// Seed value from an observed running state (first boot after
/// this store lands, or an agent spawned outside the normal path).
pub fn from_running(running: bool) -> Self {
if running { Wanted::Up } else { Wanted::Offline }
}
}
/// What a `Reconcile` should do given intent + observation. Pure so
/// the `{Up,Offline} × {up,down}` matrix is unit-testable.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReconcileAction {
Start,
Stop,
Noop,
}
#[must_use]
pub fn reconcile_action(wanted: Wanted, running: bool) -> ReconcileAction {
match (wanted, running) {
(Wanted::Up, false) => ReconcileAction::Start,
(Wanted::Offline, true) => ReconcileAction::Stop,
(Wanted::Up, true) | (Wanted::Offline, false) => ReconcileAction::Noop,
}
}
/// Sqlite-backed store. `Arc`-friendly: all methods take `&self`, the
/// internal `Mutex<Connection>` serializes access.
pub struct PowerStore {
conn: Mutex<Connection>,
}
impl PowerStore {
/// Open (a connection to) the shared coordinator DB and ensure the
/// `agent_power` table exists. `db_path` is the same sqlite file
/// the broker / approvals / questions stores open.
pub fn open(db_path: &Path) -> Result<Self> {
let conn = crate::db::open(db_path, "agent_power")?;
conn.execute_batch(SCHEMA)
.context("apply agent_power schema")?;
Ok(Self {
conn: Mutex::new(conn),
})
}
/// In-memory store for tests.
#[cfg(test)]
pub fn open_in_memory() -> Result<Self> {
let conn = Connection::open_in_memory().context("open in-memory agent_power db")?;
conn.execute_batch(SCHEMA)
.context("apply agent_power schema")?;
Ok(Self {
conn: Mutex::new(conn),
})
}
/// Read an agent's intent. `None` when the agent has no row yet
/// (callers seed from observed state via [`Self::get_or_seed`]).
pub fn get(&self, agent: &str) -> Result<Option<Wanted>> {
let conn = self.conn.lock().expect("agent_power mutex poisoned");
let row: Option<String> = conn
.query_row(
"SELECT wanted FROM agent_power WHERE agent = ?1",
params![agent],
|r| r.get(0),
)
.optional()
.context("select agent_power")?;
Ok(row.and_then(|s| Wanted::parse(&s)))
}
/// Write an agent's intent (last-writer-wins, synchronous at
/// request time).
pub fn set(&self, agent: &str, wanted: Wanted) -> Result<()> {
let conn = self.conn.lock().expect("agent_power mutex poisoned");
conn.execute(
"INSERT INTO agent_power (agent, wanted, updated_at) VALUES (?1, ?2, ?3)
ON CONFLICT(agent) DO UPDATE SET wanted = ?2, updated_at = ?3",
params![agent, wanted.as_str(), now_unix()],
)
.context("upsert agent_power")?;
Ok(())
}
/// Read an agent's intent, seeding the row from the observed
/// running state when absent — the migration rule for agents that
/// predate this store (running ⇒ `Up`, stopped ⇒ `Offline`), after
/// which the DB is authoritative.
pub fn get_or_seed(&self, agent: &str, running: bool) -> Result<Wanted> {
if let Some(w) = self.get(agent)? {
return Ok(w);
}
let seeded = Wanted::from_running(running);
self.set(agent, seeded)?;
tracing::info!(%agent, wanted = seeded.as_str(), "agent_power: seeded from observed state");
Ok(seeded)
}
/// Drop an agent's row (container destroyed).
pub fn remove(&self, agent: &str) -> Result<()> {
let conn = self.conn.lock().expect("agent_power mutex poisoned");
conn.execute("DELETE FROM agent_power WHERE agent = ?1", params![agent])
.context("delete agent_power")?;
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
/// The full `{Up,Offline} × {up,down}` reconcile matrix:
/// start / stop / noop / noop.
#[test]
fn reconcile_matrix() {
assert_eq!(reconcile_action(Wanted::Up, false), ReconcileAction::Start);
assert_eq!(
reconcile_action(Wanted::Offline, true),
ReconcileAction::Stop
);
assert_eq!(reconcile_action(Wanted::Up, true), ReconcileAction::Noop);
assert_eq!(
reconcile_action(Wanted::Offline, false),
ReconcileAction::Noop
);
}
#[test]
fn get_set_roundtrip_and_seed() {
let store = PowerStore::open_in_memory().expect("open");
assert_eq!(store.get("alice").expect("get"), None);
// Seed from observed running state, once.
assert_eq!(store.get_or_seed("alice", true).expect("seed"), Wanted::Up);
// Thereafter the DB is authoritative — observed state no longer
// overrides.
assert_eq!(
store.get_or_seed("alice", false).expect("seeded"),
Wanted::Up
);
store.set("alice", Wanted::Offline).expect("set");
assert_eq!(store.get("alice").expect("get"), Some(Wanted::Offline));
store.remove("alice").expect("remove");
assert_eq!(store.get("alice").expect("get"), None);
}
}

1155
hive-c0re/src/stores/scheduled_prompts.rs Normal file
View file

File diff suppressed because it is too large Load diff