hyperhive/hive-c0re/src/main.rs

use std::path::PathBuf;
use std::sync::Arc;

use anyhow::{Context as _, Result};
use clap::{Parser, Subcommand};

// `hive-c0re` is bin-only: this binary owns the whole daemon module
// tree. The operator CLI moved to the standalone `hivectl` crate (it
// talks to the daemon over the host admin socket), so there's no longer
// a library shared between two binaries — the modules that used to live
// in `src/lib.rs` are declared here directly.
//
// Cohesive clusters live in directory submodules (`stores`, `stats`,
// `agent_config`, `workers`); each child is re-exported at the crate
// root so `crate::broker::…` style paths keep resolving unchanged.
mod actions;
mod agent_config;
mod agent_set;
mod attic;
mod container_view;
mod coordinator;
mod dashboard;
mod dashboard_events;
mod forge;
mod gateway_nginx;
mod job_queue;
mod lifecycle;
mod loose_ends;
mod meta;
mod meta_rev;
mod migrate;
mod paths;
mod priv_client;
mod secret_file;
mod server;
mod snapshot_push;
mod socket_server;
mod stats;
mod stores;
mod swarm_agent_status;
mod swarm_notices;
mod swarm_queue;
mod swarm_status;
#[cfg(test)]
mod test_env;
mod workers;

pub(crate) use agent_config::{archetypes, limits, permissions, resource_limits, topology};
pub(crate) use stats::{container_stats, hive_stats, host_stats, otel_metrics, warnings};
pub(crate) use stores::{broker, build_logs, db, power};
pub(crate) use workers::{agent_sockets, auto_update, crash_watch, knowledge, mcp_sockets};

use coordinator::{Coordinator, HiveEnv, ServeConfig};

#[derive(Parser)]
#[command(name = "hive-c0re", about = "hyperhive coordinator daemon")]
struct Cli {
    /// Path to the host admin socket.
    #[arg(long, global = true, default_value = crate::paths::HOST_SOCKET)]
    socket: PathBuf,

    #[command(subcommand)]
    cmd: Cmd,
}

#[derive(Subcommand)]
enum Cmd {
    /// Run the coordinator daemon.
    Serve {
        /// Path to a JSON config file holding the host-level daemon config
        /// (the [`ServeConfig`] shape:
        /// the container-injected `HiveEnv` fields + the hive-c0re-local
        /// `model_prices` table). Used as the base; any per-flag override
        /// below wins over the file. Absent → start from the built-in
        /// defaults. The NixOS module passes a generated config here so the
        /// `ExecStart` stays short instead of carrying every setting — and
        /// the context-window / price-table JSON blobs — as flags.
        #[arg(long)]
        config: Option<PathBuf>,
        /// Path to the sqlite message store.
        #[arg(long, default_value = crate::paths::BROKER_DB)]
        db: PathBuf,
        /// Override: store path of the hyperhive flake source, used to
        /// identify the running revision.
        #[arg(long)]
        hyperhive_flake: Option<String>,
        /// Override: dashboard HTTP port.
        #[arg(long)]
        dashboard_port: Option<u16>,
        /// Override: systemd `CPUQuota=` applied to every agent container
        /// via a drop-in (e.g. `"200%"` = 2 cores). Set via
        /// `services.hyperhive.agentCpuQuota`.
        #[arg(long)]
        agent_cpu_quota: Option<String>,
        /// Override: systemd `MemoryMax=` applied to every agent
        /// container. Set via `services.hyperhive.agentMemoryMax`.
        #[arg(long)]
        agent_memory_max: Option<String>,
        /// Override: systemd `CPUWeight=` applied to every agent container
        /// via the same drop-in — a cgroup v2 relative share under
        /// contention (1..=10000), not a cap. Set via
        /// `services.hyperhive.agentCpuWeight`.
        #[arg(long)]
        agent_cpu_weight: Option<u32>,
        /// Override: systemd `IOWeight=` applied to every agent container,
        /// the block-IO counterpart of `--agent-cpu-weight`. Set via
        /// `services.hyperhive.agentIoWeight`.
        #[arg(long)]
        agent_io_weight: Option<u32>,
        /// Override: per-model USD prices (per million tokens) for the
        /// hive-wide ST4TS cost estimate, as a JSON object mapping a
        /// model-family short name to `{input, output, cache_read,
        /// cache_write}`. Models not covered fall back to the built-in
        /// estimate. Set via the `services.hyperhive.modelPrices` NixOS
        /// option.
        #[arg(long)]
        model_prices: Option<String>,
        /// Override: number of concurrent closure-fetching job-queue nodes
        /// (prebuild / profile-swap / create / meta lock). Set via the
        /// `services.hyperhive.deploy.hive-controller.buildSlots` NixOS option.
        #[arg(long)]
        build_slots: Option<usize>,
    },
}

#[tokio::main]
async fn main() -> Result<()> {
    hive_log::init();
    swarm_secret_client::install_crypto_provider();

    let cli = Cli::parse();
    match cli.cmd {
        Cmd::Serve {
            config,
            db,
            hyperhive_flake,
            dashboard_port,
            agent_cpu_quota,
            agent_memory_max,
            agent_cpu_weight,
            agent_io_weight,
            model_prices,
            build_slots,
        } => {
            // Base config from the --config file (or the built-in
            // defaults), then apply any per-flag overrides — config
            // file is the base, explicit flags win.
            let mut sc = match &config {
                Some(p) => {
                    let s = std::fs::read_to_string(p)
                        .with_context(|| format!("read --config {}", p.display()))?;
                    serde_json::from_str::<ServeConfig>(&s)
                        .with_context(|| format!("parse --config {}", p.display()))?
                }
                None => ServeConfig::default(),
            };
            if let Some(v) = hyperhive_flake {
                sc.env.hyperhive_flake = v;
            }
            if let Some(v) = dashboard_port {
                sc.env.dashboard_port = v;
            }
            if let Some(v) = agent_cpu_quota {
                sc.env.agent_cpu_quota = v;
            }
            if let Some(v) = agent_memory_max {
                sc.env.agent_memory_max = v;
            }
            // Passing the flag sets a weight; omitting it keeps whatever the
            // config file says (including `null` = don't emit the setting).
            // There's deliberately no flag spelling for "clear it" — that's
            // what the nix option's `null` is for.
            if let Some(v) = agent_cpu_weight {
                sc.env.agent_cpu_weight = Some(v);
            }
            if let Some(v) = agent_io_weight {
                sc.env.agent_io_weight = Some(v);
            }
            if let Some(v) = model_prices {
                sc.model_prices =
                    serde_json::from_str(&v).context("--model-prices: invalid JSON")?;
            }
            if let Some(v) = build_slots {
                sc.build_slots = v;
            }
            // Network isolation is required and its settings are
            // process-global, so a bad value breaks every container, not
            // one. Validate once here: the unit then fails visibly at
            // start with a single diagnostic naming the bad value,
            // instead of coming up "healthy" and failing each container
            // configure separately — possibly after some already
            // succeeded.
            lifecycle::network_isolation_from_env()
                .context("network isolation settings are required at startup")?;
            // The forge integration is mandatory (the NixOS module sets both
            // vars unconditionally), so check them before any worker starts.
            forge::validate_env()?;
            cmd_serve(sc.env, sc.model_prices, sc.build_slots, db, &cli.socket).await
        }
    }
}

/// One periodic sweep tick. Queues unconditionally; the node's own
/// capacity-1 resource dep (`Resource::KnowledgeTree`) is
/// what keeps two passes of the same kind from running together, so a tick
/// landing while one is live just queues one more pass behind it.
fn submit_periodic_sweep(
    coord: &Coordinator,
    kind: &job_queue::NodeKind,
    declare: impl FnOnce(&job_queue::JobBuilder) -> job_queue::Handle<'_>,
) {
    if let Err(e) = coord.job_queue.insert_job(|b| vec![declare(b).guid()]) {
        tracing::warn!(kind = <&str>::from(kind), error = ?e, "periodic sweep: submit failed");
    }
}

/// Start the coordinator daemon: open the broker, run migrations, spawn
/// background tasks (auto-update, vacuums, crash-watcher,
/// dashboard), then serve the admin socket until a signal arrives.
#[allow(
    clippy::too_many_lines,
    reason = "startup orchestration: open the broker, run migrations, then spawn \
              the full set of background services (auto-update, vacuums, \
              crash-watch, schedulers, dashboard) before the serve loop — the \
              length is inherent to booting the daemon, not the arg list"
)]
async fn cmd_serve(
    env: HiveEnv,
    model_prices: crate::hive_stats::PriceTable,
    build_slots: usize,
    db: std::path::PathBuf,
    socket: &std::path::Path,
) -> Result<()> {
    // Move any host-side state still at the legacy flat layout into its
    // subdir (`db/`, `forge/`, `run/`) BEFORE opening the
    // broker db — the broker + build-logs dbs are among the relocated
    // files. Idempotent; a no-op once migrated.
    crate::paths::relocate_legacy_state();
    // `dashboard_port` is consumed into the Coordinator below; capture the
    // Copy value first for the dashboard + knowledge-webhook tasks.
    let dashboard_port = env.dashboard_port;
    let coord = Arc::new(Coordinator::open(&db, env, model_prices, build_slots)?);
    // Idempotent pre-flight: rewrite pre-meta-layout applied
    // repos, ensure proposed repos carry the `applied`
    // remote, refresh the files that follow the agent set.
    // Runs before the boot reconcile.
    if let Err(e) = migrate::run().await {
        tracing::warn!(error = ?e, "startup migration failed");
    }
    // Before any agent starts, so the bind mounts it gets from the stored
    // permissions are the meta repo's grants until the swarm answers.
    crate::workers::permissions_import::seed_local();
    // Auto-update in the background — don't block service start.
    // Sub-agent rebuilds can take tens of seconds; we want the admin
    // socket up immediately.
    let update_coord = coord.clone();
    tokio::spawn(async move {
        if let Err(e) = auto_update::run(update_coord).await {
            tracing::warn!(error = ?e, "auto-update task failed");
        }
    });
    // Knowledge periodic pull: hourly fallback in case a knowledge event from
    // the swarm is missed (e.g. hive-c0re was down when it was sent). Sleeps
    // before its first pull: the boot `NodeKind::KnowledgePull` DAG node
    // already pulls once at startup.
    let mut knowledge_shutdown = coord.shutdown_rx();
    let knowledge_coord = coord.clone();
    tokio::spawn(async move {
        let interval = std::time::Duration::from_hours(1);
        loop {
            tokio::select! {
                () = tokio::time::sleep(interval) => submit_periodic_sweep(
                    &knowledge_coord,
                    &job_queue::NodeKind::KnowledgePull,
                    job_queue::templates::knowledge_pull,
                ),
                _ = knowledge_shutdown.changed() => {
                    tracing::info!("knowledge pull: shutdown signal received");
                    break;
                }
            }
        }
    });
    // Disk-pressure watch: raise a `disk_pressure` warning while the
    // host nix store is over threshold and clear it when back under, via the
    // push-based warnings registry. Replaces the old per-`/api/state`
    // `statvfs`. ~60s cadence; first tick fires immediately. The held guard
    // lives for the task's lifetime — dropping it (on shutdown) clears the
    // warning.
    let mut disk_shutdown = coord.shutdown_rx();
    tokio::spawn(async move {
        let mut disk_guard: Option<warnings::WarningGuard> = None;
        let interval = std::time::Duration::from_mins(1);
        loop {
            host_stats::refresh_disk_warning(&mut disk_guard);
            tokio::select! {
                () = tokio::time::sleep(interval) => {}
                _ = disk_shutdown.changed() => {
                    tracing::info!("disk-pressure watch: shutdown signal received");
                    break;
                }
            }
        }
    });
    // Periodic broker vacuum: drop fully-acked messages older
    // than 30 days. Delivered-but-unacked rows (recoverable via
    // requeue_inflight) and undelivered rows are always kept.
    // Runs hourly; first sweep happens immediately.
    let vacuum_coord = coord.clone();
    let mut vacuum_shutdown = coord.shutdown_rx();
    tokio::spawn(async move {
        let interval = std::time::Duration::from_hours(1);
        let keep_secs: i64 = 30 * 24 * 3600;
        loop {
            match vacuum_coord.broker.vacuum_delivered(keep_secs) {
                Ok(0) => {}
                Ok(n) => tracing::info!(removed = n, "broker vacuum"),
                Err(e) => tracing::warn!(error = ?e, "broker vacuum failed"),
            }
            tokio::select! {
                () = tokio::time::sleep(interval) => {}
                _ = vacuum_shutdown.changed() => {
                    tracing::info!("broker vacuum: shutdown signal received");
                    break;
                }
            }
        }
    });
    // Offer this hive's readiness to the swarm, if one is configured.
    // A no-op on a standalone hive (no queue env, logged once) — see
    // swarm_status, which owns the whole task including its own decision
    // not to start.
    swarm_status::spawn(std::sync::Arc::clone(&coord), coord.shutdown_rx());
    // Offer each locally-hosted agent's status upward too — a sibling task,
    // not a part of the above; see swarm_agent_status's module doc for why
    // the two buckets stay separate.
    swarm_agent_status::spawn(coord.shutdown_rx());
    // Squid's users file: the hashes of this hive's agents' proxy credentials.
    workers::squid_users::spawn(coord.shutdown_rx());
    // Per-agent events.sqlite + bash-tasks file cleanup now runs
    // agent-side in the harness (`hive_agent::vacuum`): the files are
    // agent-owned, so host-side deletes hit PermissionDenied / readonly-db
    // under privsep. See issue tracker "perms borked".
    // (turn-stats.sqlite has no vacuum — it's one tiny row per turn,
    // ~hundreds of KB, and the /stats + hive-stats views read it
    // directly; pruning it would just lose trend history for no gain.)
    // Slow per-container disk sampler: hive-priv measures each agent's state/
    // + work/ + writable rootfs every 30 min, cached so a sample only reads
    // cgroup files. Feeds `disk_bytes` to the OTEL exporter. See
    // container_stats::disk_sampler_loop.
    crate::container_stats::spawn_disk_sampler();
    // Per-agent container-resource OTEL export: rides the same cgroup
    // gauges out to the hive's collector on the bridge
    // (`OTEL_EXPORTER_OTLP_ENDPOINT`), unauthenticated. The swarm-otel
    // collector alone forwards upstream. No-op when the endpoint is unset.
    crate::otel_metrics::spawn_exporter(&coord.hyperhive_flake);
    // build_logs.sqlite vacuum: c0re-side (single db). Failures kept
    // 30d, successes 24h — see `build_logs::vacuum` for the rule.
    crate::build_logs::spawn_vacuum(&coord);
    // Container crash watcher: warn!()s "container crash detected" when a
    // previously-running container goes away without an operator-initiated
    // transient state — a swarm-level grafana alert matches that line.
    crash_watch::spawn(coord.clone());
    // Agent-sockets marker poll: re-fires `agent_sockets::write`
    // and `gateway_nginx::write` every 10s so the JSON and nginx
    // config pick up newly-bound `.bound` markers after a rebuild.
    // Also retries any pending gateway nginx reload that failed on
    // the previous tick. write() is idempotent so steady-state cost
    // is one stat per agent per tick.
    // See `docs/networking/gateway.md::Per-agent unix-socket upstream`.
    agent_sockets::spawn_poll();
    // MCP socket listener startup sync: one-shot sweep that re-registers any
    // running agent container whose MCP listener was lost when hive-c0re
    // restarted (Coordinator starts empty; /run/hyperhive/agents/ is tmpfs).
    // After this, listener registration is event-driven: run_create /
    // run_reconcile call register_agent on start; kill/destroy call
    // unregister_agent. No recurring poll needed — c0re owns the listeners.
    mcp_sockets::sync_on_start(coord.clone()).await;
    // Job-queue scheduler: drives the global DAG queue (rebuild /
    // spawn / power ops). Concurrency comes from the
    // build-slot count + per-agent leases inside the queue, not from
    // multiple workers — cheap nodes (graceful signals, drains,
    // reconciles) overlap nix-heavy ones structurally. Call sites
    // (auto_update, dashboard, manager) submit DAGs
    // instead of awaiting lifecycle work inline. See `job_queue/`.
    {
        let q_coord = coord.clone();
        tokio::spawn(async move {
            job_queue::scheduler::run_worker(q_coord).await;
        });
    }
    // Forward every broker event onto the unified dashboard
    // channel with a freshly-stamped seq, so the dashboard SSE
    // sees broker messages + future mutation events on one
    // stream with one monotonic seq. The broker's intra-process
    // channel (used by `recv_blocking_batch`) stays untouched.
    spawn_broker_to_dashboard_forwarder(coord.clone());
    let dash_coord = coord.clone();
    tokio::spawn(async move {
        if let Err(e) = dashboard::serve(dashboard_port, dash_coord).await {
            tracing::error!(error = ?e, "dashboard failed");
        }
    });
    // Run the admin socket until a signal arrives; then signal
    // all background tasks so they exit cleanly before the
    // process terminates.
    let coord_sig = coord.clone();
    tokio::select! {
        res = server::serve(socket, coord) => { res? }
        _ = tokio::signal::ctrl_c() => {
            tracing::info!("SIGINT received — requesting shutdown");
            coord_sig.request_shutdown();
        }
        () = async {
            let mut sig = tokio::signal::unix::signal(
                tokio::signal::unix::SignalKind::terminate()
            ).expect("failed to install SIGTERM handler");
            sig.recv().await;
        } => {
            tracing::info!("SIGTERM received — requesting shutdown");
            coord_sig.request_shutdown();
        }
    }
    Ok(())
}

/// Re-emit every broker `MessageEvent` onto the dashboard channel as
/// a `DashboardEvent::Sent` / `Delivered` with a freshly-stamped seq.
/// Background task; runs for the life of the process. On a lagged
/// broker subscription we just keep going — the dashboard channel is
/// best-effort presentation plumbing, the broker keeps its own sqlite
/// log for replay.
fn spawn_broker_to_dashboard_forwarder(coord: Arc<Coordinator>) {
    use broker::MessageEvent;
    use dashboard_events::DashboardEvent;
    let mut rx = coord.broker.subscribe();
    tokio::spawn(async move {
        loop {
            match rx.recv().await {
                Ok(MessageEvent::Sent {
                    id,
                    from,
                    to,
                    body,
                    at,
                    in_reply_to,
                }) => {
                    coord.emit_dashboard_event(DashboardEvent::Sent {
                        seq: coord.next_seq(),
                        id,
                        from,
                        to,
                        body,
                        at: hive_sh4re::wire_time::from_secs(at),
                        in_reply_to,
                    });
                }
                Ok(MessageEvent::Delivered {
                    id,
                    from,
                    to,
                    body,
                    at,
                    in_reply_to,
                }) => {
                    coord.emit_dashboard_event(DashboardEvent::Delivered {
                        seq: coord.next_seq(),
                        id,
                        from,
                        to,
                        body,
                        at: hive_sh4re::wire_time::from_secs(at),
                        in_reply_to,
                    });
                }
                Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
                    tracing::warn!(skipped = n, "broker-to-dashboard forwarder lagged");
                }
                Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
            }
        }
    });
}