90e33cbd19
The hive no longer sets an agent's CPU or memory cap: caps come from the swarm, else the hive-wide defaults. The container load tab was the only hive UI that set them, and its figures are already exported as OTEL metrics, so with the tab gone nothing wrote meta/resource-limits.json.
Removed: the file's reader and writer, the `hivectl agent set-limits` verb with its host-socket request, the `GET /api/container-resources` and `POST /api/resource-limits/{name}` dashboard endpoints, and the `cpu_quota`/`memory_max` fields of the container view that only the tab read. A leftover resource-limits.json is ignored.
405 lines
18 KiB
Rust
405 lines
18 KiB
Rust
//! Hyperhive dashboard. Lists managed containers (with deep-links to each
|
|
//! container's web UI) and the manager.
|
|
|
|
use std::net::SocketAddr;
|
|
use std::sync::Arc;
|
|
|
|
use anyhow::{Context, Result};
|
|
use axum::{
|
|
Json,
|
|
http::StatusCode,
|
|
response::{IntoResponse, Response},
|
|
routing::get,
|
|
};
|
|
use utoipa::OpenApi;
|
|
use utoipa_axum::{router::OpenApiRouter, routes};
|
|
|
|
use crate::coordinator::Coordinator;
|
|
use crate::lifecycle;
|
|
|
|
/// Root of the auto-generated `OpenAPI` spec, served raw at
|
|
/// `/api/openapi.json` — see [`utoipa`]. Swagger UI itself (browsable at
|
|
/// `/api/docs`) is nginx-hosted from the nix store (see
|
|
/// `nix/host-modules/hive-gateway/vhosts.nix`'s `swaggerUiLocations`).
|
|
/// Only routes carrying a `#[utoipa::path(...)]` annotation show up in
|
|
/// the spec — an unannotated route just doesn't appear, nothing breaks.
|
|
#[derive(OpenApi)]
|
|
#[openapi(
|
|
info(
|
|
title = "hyperhive dashboard API",
|
|
description = "hive-c0re's HTTP surface, served on the loopback \
|
|
dashboard port behind the gateway's /api/ + \
|
|
/health/ proxy prefixes."
|
|
),
|
|
tags(
|
|
(name = "health", description = "hive-wide liveness/readiness probes"),
|
|
(name = "build_logs", description = "build log headers, full rows, and raw text downloads"),
|
|
(name = "lifecycle_ops", description = "agent container lifecycle: rebuild/restart/start/stop/pause/limits"),
|
|
(name = "meta_inputs", description = "bulk flake-input update for the meta flake"),
|
|
(name = "misc_api", description = "mark-all-read, compose, spawn-request, hive stats"),
|
|
(name = "permissions", description = "tool-group + capability assignment for agents"),
|
|
(name = "rebuild_queue", description = "rebuild-queue cancel"),
|
|
(name = "state_files", description = "proxied reads of allow-listed per-agent state files"),
|
|
(name = "state_snapshot", description = "cold-load dashboard snapshot"),
|
|
(name = "tombstones", description = "purge of retained state for destroyed agents"),
|
|
)
|
|
)]
|
|
struct ApiDoc;
|
|
|
|
mod build_logs;
|
|
// The single validated identifier type — homed in `hive-host-sock` (the crate
|
|
// owning agent-path facts) so every dashboard path-param validates through the
|
|
// same type used to build agent paths. Re-exported so submodules + the socket
|
|
// server reach it as `crate::dashboard::Ident`.
|
|
pub(crate) use hive_types::Ident;
|
|
mod health;
|
|
mod lifecycle_ops;
|
|
mod meta_inputs;
|
|
mod misc_api;
|
|
pub(crate) mod permissions;
|
|
mod rebuild_queue;
|
|
mod state_files;
|
|
mod state_snapshot;
|
|
mod tombstones;
|
|
|
|
// Run after lock bumps by the job queue (`job_queue/exec.rs`); the view
|
|
// type feeds `DashboardEvent::MetaInputsChanged` (`dashboard_events.rs`).
|
|
// Re-exported to preserve the `crate::dashboard::*` paths across the split.
|
|
pub use meta_inputs::MetaInputView;
|
|
pub(crate) use meta_inputs::emit_meta_inputs_snapshot;
|
|
// Run at broker-message ingest by the coordinator + the operator-msg path
|
|
// (`main.rs`); re-exported to preserve the `crate::dashboard::scan_validated_paths`
|
|
// path across the split.
|
|
pub use state_files::scan_validated_paths;
|
|
// Called after destroy/purge/spawn finalisation (`actions.rs`); the view
|
|
// type feeds `DashboardEvent::TombstonesChanged` (`dashboard_events.rs`).
|
|
// Re-exported to preserve the `crate::dashboard::*` paths across the split.
|
|
pub use tombstones::TombstoneView;
|
|
pub(crate) use tombstones::emit_tombstones_snapshot;
|
|
|
|
#[derive(Clone)]
|
|
struct AppState {
|
|
coord: Arc<Coordinator>,
|
|
}
|
|
|
|
#[allow(
|
|
clippy::too_many_lines,
|
|
reason = "the body is dominated by the flat axum route table — one line \
|
|
per endpoint mapping a URL to its (now per-concern submodule) \
|
|
handler; splitting that exhaustive list across helpers would \
|
|
obscure the route map for no readability gain"
|
|
)]
|
|
fn app(coord: Arc<Coordinator>) -> axum::Router {
|
|
// API-only: the gateway static-serves the dashboard dist and proxies
|
|
// non-static requests here (see `nix/host-modules/hive-gateway/vhosts.nix`).
|
|
// Unmatched paths 404.
|
|
// No static fallback needed here either. Every route (SSE streams
|
|
// included — `utoipa::path` can document an SSE body as an opaque
|
|
// `text/event-stream` string, it just can't model individual frame
|
|
// shapes) is registered below via `OpenApiRouter`; see the `router`/
|
|
// `api` split.
|
|
//
|
|
// `routes!()` folds every handler it's given into ONE shared
|
|
// `MethodRouter` for the whole macro invocation — it does NOT group
|
|
// by path internally, so passing it handlers for more than one
|
|
// distinct path panics at runtime ("Overlapping method route")
|
|
// the moment two of them share an HTTP method, which most
|
|
// GET-vs-GET or POST-vs-POST pairs across different paths do. Each
|
|
// call below is therefore scoped to exactly one path — two handlers
|
|
// in the same call only when they genuinely share a path with
|
|
// different methods — chained via
|
|
// repeated `.routes(...)` calls instead of one giant `routes!(...)`
|
|
// with everything in it.
|
|
let (router, api) = OpenApiRouter::<AppState>::with_openapi(ApiDoc::openapi())
|
|
.routes(routes!(health::get_health_live))
|
|
.routes(routes!(health::get_health_ready))
|
|
.routes(routes!(state_snapshot::api_state))
|
|
.routes(routes!(state_files::get_state_file))
|
|
.routes(routes!(misc_api::api_stats_hive))
|
|
.routes(routes!(misc_api::post_mark_all_read))
|
|
.routes(routes!(misc_api::post_op_send))
|
|
.routes(routes!(build_logs::get_build_logs_all))
|
|
.routes(routes!(build_logs::get_build_log_for_node))
|
|
.routes(routes!(build_logs::get_build_log_raw_for_node))
|
|
.routes(routes!(build_logs::get_build_logs_agent))
|
|
.routes(routes!(build_logs::get_build_log_full))
|
|
.routes(routes!(build_logs::get_build_log_raw))
|
|
.routes(routes!(permissions::get_tool_groups))
|
|
.routes(routes!(permissions::post_tool_groups))
|
|
.routes(routes!(permissions::get_capabilities))
|
|
.routes(routes!(permissions::post_capabilities))
|
|
.routes(routes!(permissions::post_permissions))
|
|
.routes(routes!(permissions::get_stale_permissions))
|
|
.routes(routes!(permissions::delete_agent_permissions))
|
|
.routes(routes!(rebuild_queue::post_rebuild_queue_cancel))
|
|
.routes(routes!(lifecycle_ops::post_destroy))
|
|
.routes(routes!(lifecycle_ops::post_kill))
|
|
.routes(routes!(lifecycle_ops::post_restart))
|
|
.routes(routes!(lifecycle_ops::post_start))
|
|
.routes(routes!(lifecycle_ops::post_rebuild))
|
|
.routes(routes!(lifecycle_ops::post_pause))
|
|
.routes(routes!(lifecycle_ops::post_resume))
|
|
.routes(routes!(lifecycle_ops::post_update_all))
|
|
.routes(routes!(tombstones::post_purge_tombstone))
|
|
.routes(routes!(meta_inputs::post_meta_update))
|
|
.routes(routes!(build_logs::get_build_log_stream))
|
|
.routes(routes!(state_snapshot::dashboard_stream))
|
|
.routes(routes!(state_snapshot::dashboard_history))
|
|
.routes(routes!(state_snapshot::jobq_graph))
|
|
.routes(routes!(state_snapshot::jobq_rollup))
|
|
.split_for_parts();
|
|
// Just the JSON, not the UI — Swagger UI itself is nginx-hosted from
|
|
// the nix store (see the module doc comment above `ApiDoc`). `api`
|
|
// is `Clone`; each request gets its own owned copy for `Json` to
|
|
// serialize.
|
|
router
|
|
.route(
|
|
"/api/openapi.json",
|
|
get(move || async move { Json(api.clone()) }),
|
|
)
|
|
.with_state(AppState { coord })
|
|
}
|
|
|
|
pub async fn serve(port: u16, coord: Arc<Coordinator>) -> Result<()> {
|
|
// Binds loopback-only; external access via gateway.
|
|
// Rationale: docs/networking/gateway.md::Firewall posture.
|
|
let addr = SocketAddr::from(([127, 0, 0, 1], port));
|
|
let listener = bind_with_retry(addr).await?;
|
|
tracing::info!(%addr, "dashboard listening");
|
|
axum::serve(listener, app(coord)).await?;
|
|
Ok(())
|
|
}
|
|
|
|
// SPA shape + SSE channels: docs/web-ui/shape.md.
|
|
|
|
/// `SO_REUSEADDR` bind with retry. Retry mechanics, attempt-cap
|
|
/// rationale, and log-level cadence: `docs/web-ui/shape.md::Listener bind`.
|
|
async fn bind_with_retry(addr: SocketAddr) -> Result<tokio::net::TcpListener> {
|
|
let mut delay_ms = 250u64;
|
|
let mut attempts = 0u32;
|
|
loop {
|
|
match try_bind(addr) {
|
|
Ok(l) => {
|
|
if attempts > 0 {
|
|
tracing::info!(
|
|
%addr, attempts,
|
|
"dashboard: bind succeeded after retry"
|
|
);
|
|
}
|
|
return Ok(l);
|
|
}
|
|
Err(e) if e.kind() == std::io::ErrorKind::AddrInUse => {
|
|
let attempt = attempts + 1;
|
|
if attempt <= 12 {
|
|
tracing::warn!(
|
|
%addr, attempt,
|
|
"dashboard: AddrInUse, retrying in {delay_ms}ms"
|
|
);
|
|
} else {
|
|
tracing::info!(
|
|
%addr, attempt,
|
|
"dashboard: AddrInUse still holding, retrying in {delay_ms}ms"
|
|
);
|
|
}
|
|
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
|
|
attempts += 1;
|
|
delay_ms = (delay_ms * 2).min(2000);
|
|
}
|
|
Err(e) => {
|
|
return Err(e).with_context(|| format!("bind dashboard on {addr}"));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
fn try_bind(addr: SocketAddr) -> std::io::Result<tokio::net::TcpListener> {
|
|
let sock = match addr {
|
|
SocketAddr::V4(_) => tokio::net::TcpSocket::new_v4()?,
|
|
SocketAddr::V6(_) => tokio::net::TcpSocket::new_v6()?,
|
|
};
|
|
sock.set_reuseaddr(true)?;
|
|
sock.bind(addr)?;
|
|
sock.listen(1024)
|
|
}
|
|
|
|
/// Two-axis path-param guard for write routes. Combines:
|
|
///
|
|
/// 1. **format validation** ([`Ident::parse`]) — rejects
|
|
/// path traversal / unicode homoglyphs / empty + too-long names with
|
|
/// HTTP 400.
|
|
/// 2. **existence check** — looks up `name` in the coordinator's
|
|
/// container snapshot; unknown name → HTTP 404 with a clear
|
|
/// "no such agent" message. catches the operator-typo case where
|
|
/// a destructive POST would otherwise hit silently (mark-all-read
|
|
/// returning 0) or hit downstream lifecycle code that fails with
|
|
/// a confusing nspawn error.
|
|
///
|
|
/// Returns `None` when both checks pass (caller proceeds), `Some(Response)`
|
|
/// when the request should be rejected. Use at the top of every write
|
|
/// handler taking a name path-param. Read-only GET handlers and
|
|
/// handlers that legitimately operate on tombstoned agents (e.g.
|
|
/// `mark-all-read` on broker rows for a destroyed agent) call
|
|
/// [`Ident::parse`] directly and skip the existence check.
|
|
async fn guard_agent_name(state: &AppState, name: &str) -> Option<Response> {
|
|
if let Err(reason) = Ident::parse(name) {
|
|
return Some(
|
|
(StatusCode::BAD_REQUEST, format!("bad agent name: {reason}")).into_response(),
|
|
);
|
|
}
|
|
let snapshot = state.coord.containers_snapshot().await;
|
|
if !snapshot.iter().any(|c| c.name == name) {
|
|
return Some((StatusCode::NOT_FOUND, format!("no such agent: {name}")).into_response());
|
|
}
|
|
None
|
|
}
|
|
|
|
/// Convert either a logical name or a container name back to the logical
|
|
/// name. Sub-agents are `h-foo` → `foo`; manager stays `root`.
|
|
fn strip_container_prefix(name: &str) -> String {
|
|
name.strip_prefix(lifecycle::AGENT_PREFIX)
|
|
.unwrap_or(name)
|
|
.to_owned()
|
|
}
|
|
|
|
/// The common internal-error case as a `ProblemDetails`: a 500 RFC 9457
|
|
/// (`application/problem+json`) value via the `problem_details` crate.
|
|
/// `from_status_code` sets `status` + `title` (the canonical reason phrase)
|
|
/// and leaves `type` as the default `about:blank`; `with_detail` carries the
|
|
/// caller message; the crate's axum `IntoResponse` emits the
|
|
/// `application/problem+json` body the frontend parses (it reads `detail`).
|
|
/// Handlers that surface client failures return `Result<_, ProblemDetails>`
|
|
/// and hand this (or an inline `from_status_code(4xx)`) straight to `Err` —
|
|
/// no manual `.into_response()`.
|
|
fn error_problem(message: &str) -> problem_details::ProblemDetails {
|
|
problem_details::ProblemDetails::from_status_code(StatusCode::INTERNAL_SERVER_ERROR)
|
|
.with_detail(message)
|
|
}
|
|
|
|
/// `Response` wrapper around [`error_problem`] for the many handlers typed
|
|
/// `-> Response` whose only failure mode is a 500 — they funnel errors
|
|
/// through here rather than threading a `Result` return type.
|
|
fn error_response(message: &str) -> Response {
|
|
error_problem(message).into_response()
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
/// Status code of a bare HTTP/1.1 GET against the dashboard's own route
|
|
/// table, served on an ephemeral loopback port.
|
|
async fn status_of(path: &str) -> u16 {
|
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
|
|
let dir = tempfile::tempdir().expect("tempdir");
|
|
let coord = Coordinator::open(
|
|
&dir.path().join("broker.sqlite"),
|
|
crate::coordinator::HiveEnv::default(),
|
|
crate::hive_stats::PriceTable::new(),
|
|
1,
|
|
)
|
|
.expect("open coordinator");
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
|
.await
|
|
.expect("bind");
|
|
let addr = listener.local_addr().expect("local addr");
|
|
let server = tokio::spawn(axum::serve(listener, app(Arc::new(coord))).into_future());
|
|
let mut stream = tokio::net::TcpStream::connect(addr).await.expect("connect");
|
|
stream
|
|
.write_all(
|
|
format!("GET {path} HTTP/1.1\r\nHost: t\r\nConnection: close\r\n\r\n").as_bytes(),
|
|
)
|
|
.await
|
|
.expect("write");
|
|
let mut reply = String::new();
|
|
stream.read_to_string(&mut reply).await.expect("read");
|
|
server.abort();
|
|
reply
|
|
.split_whitespace()
|
|
.nth(1)
|
|
.and_then(|code| code.parse().ok())
|
|
.unwrap_or_else(|| panic!("no status line in {reply:?}"))
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn operator_inbox_route_is_gone() {
|
|
assert_eq!(status_of("/health/live").await, 200);
|
|
assert_eq!(status_of("/api/operator-inbox").await, 404);
|
|
}
|
|
|
|
#[test]
|
|
fn problem_details_carry_rfc9457_status_and_detail() {
|
|
// Contract the frontend depends on: the problem_details crate
|
|
// serialises the RFC 9457 members we rely on — `status` (numeric)
|
|
// and `detail` (the caller message; the FE reads `.detail`).
|
|
let pd = problem_details::ProblemDetails::from_status_code(StatusCode::BAD_REQUEST)
|
|
.with_detail("bad input");
|
|
let v = serde_json::to_value(&pd).expect("problem details serialise");
|
|
assert_eq!(v["status"], 400);
|
|
assert_eq!(v["detail"], "bad input");
|
|
// The 500 wrapper path carries the internal-error status.
|
|
let five =
|
|
problem_details::ProblemDetails::from_status_code(StatusCode::INTERNAL_SERVER_ERROR)
|
|
.with_detail("boom");
|
|
let fv = serde_json::to_value(&five).expect("problem details serialise");
|
|
assert_eq!(fv["status"], 500);
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod router_build_probe {
|
|
use super::*;
|
|
|
|
/// Regression test for the exact panic this file's `serve()` used to
|
|
/// hit at startup: `routes!()` folds every handler passed to ONE
|
|
/// macro invocation into a single shared `MethodRouter`, so two
|
|
/// handlers on different paths but the same HTTP method panic with
|
|
/// "Overlapping method route" the moment they're in the same
|
|
/// `routes!(...)` call — see `serve()`'s comment above its own
|
|
/// `.routes(...)` chain. This mirrors that exact chain (one
|
|
/// `.routes()` call per distinct path, multi-method pairs grouped
|
|
/// only where they share a path) so a future regression that
|
|
/// accidentally merges two single-path calls back into one
|
|
/// multi-path `routes!(...)` call fails here instead of at daemon
|
|
/// startup.
|
|
#[test]
|
|
fn probe_multi_path_routes_macro_does_not_panic() {
|
|
let _ = OpenApiRouter::<AppState>::with_openapi(ApiDoc::openapi())
|
|
.routes(routes!(health::get_health_live))
|
|
.routes(routes!(health::get_health_ready))
|
|
.routes(routes!(state_snapshot::api_state))
|
|
.routes(routes!(state_files::get_state_file))
|
|
.routes(routes!(misc_api::api_stats_hive))
|
|
.routes(routes!(misc_api::post_mark_all_read))
|
|
.routes(routes!(misc_api::post_op_send))
|
|
.routes(routes!(build_logs::get_build_logs_all))
|
|
.routes(routes!(build_logs::get_build_log_for_node))
|
|
.routes(routes!(build_logs::get_build_log_raw_for_node))
|
|
.routes(routes!(build_logs::get_build_logs_agent))
|
|
.routes(routes!(build_logs::get_build_log_full))
|
|
.routes(routes!(build_logs::get_build_log_raw))
|
|
.routes(routes!(permissions::get_tool_groups))
|
|
.routes(routes!(permissions::post_tool_groups))
|
|
.routes(routes!(permissions::get_capabilities))
|
|
.routes(routes!(permissions::post_capabilities))
|
|
.routes(routes!(permissions::post_permissions))
|
|
.routes(routes!(permissions::get_stale_permissions))
|
|
.routes(routes!(permissions::delete_agent_permissions))
|
|
.routes(routes!(rebuild_queue::post_rebuild_queue_cancel))
|
|
.routes(routes!(lifecycle_ops::post_destroy))
|
|
.routes(routes!(lifecycle_ops::post_kill))
|
|
.routes(routes!(lifecycle_ops::post_restart))
|
|
.routes(routes!(lifecycle_ops::post_start))
|
|
.routes(routes!(lifecycle_ops::post_rebuild))
|
|
.routes(routes!(lifecycle_ops::post_pause))
|
|
.routes(routes!(lifecycle_ops::post_resume))
|
|
.routes(routes!(lifecycle_ops::post_update_all))
|
|
.routes(routes!(tombstones::post_purge_tombstone))
|
|
.routes(routes!(meta_inputs::post_meta_update))
|
|
.routes(routes!(build_logs::get_build_log_stream))
|
|
.routes(routes!(state_snapshot::dashboard_stream))
|
|
.routes(routes!(state_snapshot::dashboard_history))
|
|
.routes(routes!(state_snapshot::jobq_graph))
|
|
.routes(routes!(state_snapshot::jobq_rollup));
|
|
}
|
|
}
|