use std::path::PathBuf; use std::sync::Arc; use anyhow::{Context as _, Result, bail}; use clap::{Parser, Subcommand}; use hive_sh4re::{HostRequest, HostResponse}; // Every module hangs off the `hive_c0re` library (see `src/lib.rs`). // The daemon and the `hivectl` sibling binary share the same module // tree — no per-binary duplication. Enumerated rather than wildcard // so clippy stays happy + the lib surface this bin consumes is // explicit (any new daemon entry point reads off the next add). use hive_c0re::coordinator::{Coordinator, HiveEnv, ServeConfig}; use hive_c0re::{ agent_sockets, auto_update, broker, client, crash_watch, dashboard, dashboard_events, forge, knowledge, matrix, migrate, rebuild_queue, reminder_scheduler, scheduled_prompts_worker, server, socket_server, }; #[derive(Parser)] #[command(name = "hive-c0re", about = "hyperhive coordinator daemon and CLI")] struct Cli { /// Path to the host admin socket. #[arg(long, global = true, default_value = "/run/hyperhive/host.sock")] socket: PathBuf, #[command(subcommand)] cmd: Cmd, } #[derive(Subcommand)] enum Cmd { /// Run the coordinator daemon. Serve { /// Path to a JSON config file holding the host-level daemon config /// (the [`ServeConfig`](hive_c0re::coordinator::ServeConfig) shape: /// the container-injected `HiveEnv` fields + the hive-c0re-local /// `model_prices` table). Used as the base; any per-flag override /// below wins over the file. Absent → start from the built-in /// defaults. The NixOS module passes a generated config here so the /// `ExecStart` stays short instead of carrying every setting — and /// the context-window / price-table JSON blobs — as flags. #[arg(long)] config: Option, /// Path to the sqlite message store. #[arg(long, default_value = hive_c0re::paths::BROKER_DB)] db: PathBuf, /// Override: URL of the hyperhive flake. Inlined into each /// per-agent `flake.nix` as the `hyperhive` input. #[arg(long)] hyperhive_flake: Option, /// Override: store-path URL of the nixpkgs to wire into the meta /// flake as `inputs.nixpkgs.url`. Empty = legacy /// `follows = "hyperhive/nixpkgs"` fallback. #[arg(long)] nixpkgs_flake: Option, /// Override: dashboard HTTP port. #[arg(long)] dashboard_port: Option, /// Override: operator pronouns (free text). Threaded into each /// container's harness via `HIVE_OPERATOR_PRONOUNS`. #[arg(long)] operator_pronouns: Option, /// Override: per-model context-window sizes, as a JSON object /// mapping model-family short name to token count. Threaded into /// each container as `HIVE_CONTEXT_WINDOW_TOKENS_` env /// vars. Set via the `services.hive-c0re.contextWindowTokens` /// NixOS option. #[arg(long)] context_window_tokens: Option, /// Override: systemd `CPUQuota=` applied to every agent container /// via a drop-in (e.g. `"200%"` = 2 cores). Set via /// `services.hyperhive.agentCpuQuota`. #[arg(long)] agent_cpu_quota: Option, /// Override: systemd `MemoryMax=` applied to every agent /// container. Set via `services.hyperhive.agentMemoryMax`. #[arg(long)] agent_memory_max: Option, /// Override: per-model USD prices (per million tokens) for the /// hive-wide ST4TS cost estimate, as a JSON object mapping a /// model-family short name to `{input, output, cache_read, /// cache_write}`. Models not covered fall back to the built-in /// estimate. Set via the `services.hyperhive.modelPrices` NixOS /// option. #[arg(long)] model_prices: Option, }, /// Spawn a new agent container directly (`hive-agent-`). Bypasses /// the approval queue — use only as an operator on the host. For /// approval-gated spawns, use `request-spawn` instead. Spawn { name: String }, /// Queue a spawn request as an approval. The container is created on /// `approve ` (CLI) or the dashboard's APPR0VE button. RequestSpawn { name: String }, /// Stop a managed container (graceful). Kill { name: String }, /// Tear down a sub-agent container. Container is removed; persistent /// state (config repos + Claude credentials) is kept by default. Pass /// `--purge` to also wipe the agent's state dirs (config + creds + /// notes). No undo. Destroy { name: String, #[arg(long)] purge: bool, }, /// Apply pending config to a managed container. Rebuild { name: String }, /// List managed containers. List, /// List pending approval requests submitted by the manager. Pending, /// Approve a pending request by id; the action runs immediately. Approve { id: i64 }, /// Deny a pending request by id. Deny { id: i64 }, /// Move an agent in the topology tree. Set `--parent` to a new /// parent agent name; pass `--root` to promote the agent to root /// (no parent). Refuses cycles and unknown agents. The manager /// is reparentable like any other agent — its privileges come /// from the privileged MCP socket, not its tree position. SetParent { child: String, /// New parent agent name. Mutually exclusive with `--root`. /// Exactly one of `--parent` / `--root` is required — clap /// rejects both-absent calls so a fat-fingered /// `hive-c0re set-parent alice` doesn't silently promote /// alice to root. #[arg(long, conflicts_with = "root", required_unless_present = "root")] parent: Option, /// Promote `child` to root (no parent). #[arg(long)] root: bool, }, } #[tokio::main] async fn main() -> Result<()> { tracing_subscriber::fmt() .with_env_filter( tracing_subscriber::EnvFilter::try_from_default_env() .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), ) .init(); let cli = Cli::parse(); match cli.cmd { Cmd::Serve { config, db, hyperhive_flake, nixpkgs_flake, dashboard_port, operator_pronouns, context_window_tokens, agent_cpu_quota, agent_memory_max, model_prices, } => { // Base config from the --config file (or the built-in // defaults), then apply any per-flag overrides — config // file is the base, explicit flags win. let mut sc = match &config { Some(p) => { let s = std::fs::read_to_string(p) .with_context(|| format!("read --config {}", p.display()))?; serde_json::from_str::(&s) .with_context(|| format!("parse --config {}", p.display()))? } None => ServeConfig::default(), }; if let Some(v) = hyperhive_flake { sc.env.hyperhive_flake = v; } if let Some(v) = nixpkgs_flake { sc.env.nixpkgs_flake = v; } if let Some(v) = dashboard_port { sc.env.dashboard_port = v; } if let Some(v) = operator_pronouns { sc.env.operator_pronouns = v; } if let Some(v) = context_window_tokens { sc.env.context_window_tokens = serde_json::from_str(&v).context("--context-window-tokens: invalid JSON")?; } if let Some(v) = agent_cpu_quota { sc.env.agent_cpu_quota = v; } if let Some(v) = agent_memory_max { sc.env.agent_memory_max = v; } if let Some(v) = model_prices { sc.model_prices = serde_json::from_str(&v).context("--model-prices: invalid JSON")?; } cmd_serve(sc.env, sc.model_prices, db, &cli.socket).await } Cmd::Spawn { name } => { render(client::request(&cli.socket, HostRequest::Spawn { name }).await?) } Cmd::RequestSpawn { name } => { render(client::request(&cli.socket, HostRequest::RequestSpawn { name }).await?) } Cmd::Kill { name } => { render(client::request(&cli.socket, HostRequest::Kill { name }).await?) } Cmd::Destroy { name, purge } => { render(client::request(&cli.socket, HostRequest::Destroy { name, purge }).await?) } Cmd::Rebuild { name } => { render(client::request(&cli.socket, HostRequest::Rebuild { name }).await?) } Cmd::List => render(client::request(&cli.socket, HostRequest::List).await?), Cmd::Pending => render(client::request(&cli.socket, HostRequest::Pending).await?), Cmd::Approve { id } => { render(client::request(&cli.socket, HostRequest::Approve { id }).await?) } Cmd::Deny { id } => render(client::request(&cli.socket, HostRequest::Deny { id }).await?), Cmd::SetParent { child, parent, root, } => { let new_parent = if root { None } else { parent }; render( client::request(&cli.socket, HostRequest::SetParent { child, new_parent }).await?, ) } } } /// Start the coordinator daemon: open the broker, run migrations, spawn /// background tasks (auto-update, vacuums, crash-watcher, reminder-scheduler, /// dashboard), then serve the admin socket until a signal arrives. #[allow( clippy::too_many_lines, reason = "startup orchestration: open the broker, run migrations, then spawn \ the full set of background services (auto-update, vacuums, \ crash-watch, schedulers, dashboard) before the serve loop — the \ length is inherent to booting the daemon, not the arg list" )] async fn cmd_serve( env: HiveEnv, model_prices: hive_c0re::hive_stats::PriceTable, db: std::path::PathBuf, socket: &std::path::Path, ) -> Result<()> { // Move any host-side state still at the legacy flat layout into its // subdir (`db/`, `forge/`, `matrix/`, `run/`) BEFORE opening the // broker db — the broker + build-logs dbs are among the relocated // files. Idempotent; a no-op once migrated. hive_c0re::paths::relocate_legacy_state(); // `dashboard_port` is consumed into the Coordinator below; capture the // Copy value first for the dashboard + knowledge-webhook tasks. let dashboard_port = env.dashboard_port; let coord = Arc::new(Coordinator::open(&db, env, model_prices)?); socket_server::start_manager(coord.clone())?; // Idempotent pre-flight: rewrite pre-meta-layout applied // repos, ensure proposed repos carry the `applied` // remote, bootstrap the meta repo, repoint containers at // `meta#` (one-shot, guarded by a marker file). // Runs before manager auto-spawn so the new manager is // built against meta from the first attempt. if let Err(e) = migrate::run(&coord).await { tracing::warn!(error = ?e, "startup migration failed"); } // Auto-create the root agent container if it isn't there yet. Block // on this — without root the system has no manager harness. // Failures are logged but allowed: a broken auto-spawn shouldn't // make the dashboard unreachable for debugging. if let Err(e) = auto_update::ensure_root_agent(&coord).await { tracing::warn!(error = ?e, "auto-spawn root agent failed"); } // Auto-update in the background — don't block service start. // Sub-agent rebuilds can take tens of seconds; we want the admin // socket up immediately. let update_coord = coord.clone(); tokio::spawn(async move { if let Err(e) = auto_update::run(update_coord).await { tracing::warn!(error = ?e, "auto-update task failed"); } }); // Forge user sweep: ensure every existing container has a // forgejo user + access token. No-op when the hive-forge // container isn't running. Backgrounded — touches the // forge state dir via `nixos-container run` which is slow. tokio::spawn(async move { forge::ensure_all().await; }); // Knowledge webhook setup: ensure the Forgejo push webhook for // `internal/knowledge` exists so `pull()` fires on merge. Runs // after forge::ensure_all so the core token + repo are present. // No-op when the core token or forge are absent. let webhook_port = dashboard_port; tokio::spawn(async move { if let Some(token) = forge::core_token() && let Err(e) = knowledge::ensure_webhook(&token, webhook_port).await { tracing::warn!(error = ?e, "knowledge: ensure_webhook failed"); } }); // Knowledge periodic pull: hourly fallback in case the webhook is // missed (e.g. hive-c0re was down during a push). First fires at // startup (immediate pull after the clone is already present). let mut knowledge_shutdown = coord.shutdown_rx(); tokio::spawn(async move { // Initial pull — reconcile any commits that landed while c0re // was offline. if let Err(e) = knowledge::pull().await { tracing::debug!(error = ?e, "knowledge: startup pull skipped (no clone yet?)"); } let interval = std::time::Duration::from_hours(1); loop { tokio::select! { () = tokio::time::sleep(interval) => { if let Err(e) = knowledge::pull().await { tracing::warn!(error = ?e, "knowledge: periodic pull failed"); } } _ = knowledge_shutdown.changed() => { tracing::info!("knowledge pull: shutdown signal received"); break; } } } }); // Matrix user sweep: same shape — ensure every container has // an account on the local matrix-tuwunel homeserver with an // access_token persisted to `/matrix-token`. No-op when // the hive-matrix container isn't running. Backgrounded because // UIAA is a two-roundtrip dance per agent. // // Runs once at startup AND periodically every 30 minutes so that // token files deleted by `hive-matrix-daemon` (stale-token // recovery — `M_UNKNOWN_TOKEN`) get re-provisioned without // requiring a hive-c0re restart. let mut matrix_shutdown = coord.shutdown_rx(); tokio::spawn(async move { let interval = std::time::Duration::from_mins(30); matrix::ensure_all().await; loop { tokio::select! { () = tokio::time::sleep(interval) => { matrix::ensure_all().await; } _ = matrix_shutdown.changed() => { tracing::info!("matrix ensure_all: shutdown signal received"); break; } } } }); // Periodic broker vacuum: drop fully-acked messages older // than 30 days. Delivered-but-unacked rows (recoverable via // requeue_inflight) and undelivered rows are always kept. // Runs hourly; first sweep happens immediately. let vacuum_coord = coord.clone(); let mut vacuum_shutdown = coord.shutdown_rx(); tokio::spawn(async move { let interval = std::time::Duration::from_hours(1); let keep_secs: i64 = 30 * 24 * 3600; loop { match vacuum_coord.broker.vacuum_delivered(keep_secs) { Ok(0) => {} Ok(n) => tracing::info!(removed = n, "broker vacuum"), Err(e) => tracing::warn!(error = ?e, "broker vacuum failed"), } tokio::select! { () = tokio::time::sleep(interval) => {} _ = vacuum_shutdown.changed() => { tracing::info!("broker vacuum: shutdown signal received"); break; } } } }); // Per-agent events.sqlite + bash-tasks file cleanup now runs // agent-side in the harness (`hive_ag3nt::vacuum`): the files are // agent-owned, so host-side deletes hit PermissionDenied / readonly-db // under privsep. See issue tracker "perms borked". // (turn-stats.sqlite has no vacuum — it's one tiny row per turn, // ~hundreds of KB, and the /stats + hive-stats views read it // directly; pruning it would just lose trend history for no gain.) // Slow per-container disk sampler: a `du` of each agent's state dir // + writable rootfs every ~5 min, cached so the 5s container-load // poll stays cheap cgroup-only reads. Feeds `disk_bytes` on the LOAD // tab. See container_stats::disk_sampler_loop. hive_c0re::container_stats::spawn_disk_sampler(); // build_logs.sqlite vacuum: c0re-side (single db). Failures kept // 30d, successes 24h — see `build_logs::vacuum` for the rule. hive_c0re::build_logs::spawn_vacuum(&coord); // audit_log.sqlite vacuum: agent-initiated privileged-action trail, // 90d retention — see `audit_log::vacuum`. hive_c0re::audit_log::spawn_vacuum(&coord); // Container crash watcher: emits HelperEvent::ContainerCrash // when a previously-running container goes away without an // operator-initiated transient state. crash_watch::spawn(coord.clone()); // Agent-sockets marker poll: re-fires `agent_sockets::write` // and `gateway_nginx::write` every 10s so the JSON and nginx // config pick up newly-bound `.bound` markers after a rebuild. // Also retries any pending gateway nginx reload that failed on // the previous tick. write() is idempotent so steady-state cost // is one stat per agent per tick. // See `docs/gateway.md::Per-agent unix-socket upstream`. agent_sockets::spawn_poll(); // Reminder scheduler: drains due reminders + handles // file_path payload persistence. See reminder_scheduler.rs. reminder_scheduler::spawn(coord.clone()); // Scheduled-prompts worker: drains due scheduled_prompts rows // and fans the body out to each active target's inbox. See // scheduled_prompts_worker.rs. scheduled_prompts_worker::spawn(coord.clone()); // Rebuild-queue worker: drains the global rebuild/meta-update/ // spawn queue FIFO so hive-c0re never runs two heavyweight // container ops concurrently. Existing rebuild call sites // (auto_update, dashboard, manager, approval handler) enqueue // here instead of awaiting `rebuild_agent` inline. See // `rebuild_queue.rs`. { let q_coord = coord.clone(); tokio::spawn(async move { rebuild_queue::run_worker(q_coord).await; }); // Fast lane: a second serial worker for hard Start/Stop, running // concurrently with the build worker above so a stop/start never // waits behind a slow build for another container. Per-agent // ordering vs that agent's own build is enforced in the queue's // claim logic (a fast op defers behind its agent's running build). let fast_coord = coord.clone(); tokio::spawn(async move { rebuild_queue::run_fast_worker(fast_coord).await; }); } // Forward every broker event onto the unified dashboard // channel with a freshly-stamped seq, so the dashboard SSE // sees broker messages + future mutation events on one // stream with one monotonic seq. The broker's intra-process // channel (used by `recv_blocking_batch`) stays untouched. spawn_broker_to_dashboard_forwarder(coord.clone()); let dash_coord = coord.clone(); tokio::spawn(async move { if let Err(e) = dashboard::serve(dashboard_port, dash_coord).await { tracing::error!(error = ?e, "dashboard failed"); } }); // Run the admin socket until a signal arrives; then signal // all background tasks so they exit cleanly before the // process terminates. let coord_sig = coord.clone(); tokio::select! { res = server::serve(socket, coord) => { res? } _ = tokio::signal::ctrl_c() => { tracing::info!("SIGINT received — requesting shutdown"); coord_sig.request_shutdown(); } () = async { let mut sig = tokio::signal::unix::signal( tokio::signal::unix::SignalKind::terminate() ).expect("failed to install SIGTERM handler"); sig.recv().await; } => { tracing::info!("SIGTERM received — requesting shutdown"); coord_sig.request_shutdown(); } } Ok(()) } /// Re-emit every broker `MessageEvent` onto the dashboard channel as /// a `DashboardEvent::Sent` / `Delivered` with a freshly-stamped seq. /// Background task; runs for the life of the process. On a lagged /// broker subscription we just keep going — the dashboard channel is /// best-effort presentation plumbing, the broker keeps its own sqlite /// log for replay. fn spawn_broker_to_dashboard_forwarder(coord: Arc) { use broker::MessageEvent; use dashboard_events::DashboardEvent; let mut rx = coord.broker.subscribe(); tokio::spawn(async move { loop { match rx.recv().await { Ok(MessageEvent::Sent { id, from, to, body, at, in_reply_to, }) => { let file_refs = dashboard::scan_validated_paths(&body); coord.emit_dashboard_event(DashboardEvent::Sent { seq: coord.next_seq(), id, from, to, body, at, in_reply_to, file_refs, }); } Ok(MessageEvent::Delivered { id, from, to, body, at, in_reply_to, }) => { let file_refs = dashboard::scan_validated_paths(&body); coord.emit_dashboard_event(DashboardEvent::Delivered { seq: coord.next_seq(), id, from, to, body, at, in_reply_to, file_refs, }); } // Transient pings are not persisted and not shown in the // dashboard message history — ignore silently. Ok(MessageEvent::Ping { .. }) => {} Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => { tracing::warn!(skipped = n, "broker-to-dashboard forwarder lagged"); } Err(tokio::sync::broadcast::error::RecvError::Closed) => break, } } }); } fn render(resp: HostResponse) -> Result<()> { println!("{}", serde_json::to_string_pretty(&resp)?); if !resp.ok { bail!(resp.error.unwrap_or_else(|| "request failed".to_owned())); } Ok(()) }