hive-c0re: the container stats comments and docs name the OTEL exporter as the consumer, not the removed tab
This commit is contained in:
parent
23cedf3fa8
commit
c875223db7
5 changed files with 16 additions and 15 deletions
|
|
@ -548,7 +548,8 @@ wants it.
|
||||||
Two things to know about the weights:
|
Two things to know about the weights:
|
||||||
|
|
||||||
- They're **hive-wide** — unlike the caps there is no per-agent
|
- They're **hive-wide** — unlike the caps there is no per-agent
|
||||||
setting, so every agent carries the same value and the weight does _not_ rank agents against each other.
|
setting, so every agent carries the same value and the weight does _not_
|
||||||
|
rank agents against each other.
|
||||||
What `80` buys is that agents yield to everything **not** on this
|
What `80` buys is that agents yield to everything **not** on this
|
||||||
drop-in path: host services (nginx and dnsmasq among them) and the
|
drop-in path: host services (nginx and dnsmasq among them) and the
|
||||||
infra containers (`hive-forge`, `hive-gateway`, `hive-matrix`), which
|
infra containers (`hive-forge`, `hive-gateway`, `hive-matrix`), which
|
||||||
|
|
|
||||||
|
|
@ -291,8 +291,8 @@ cleanly and the affected level button goes on returning zero rows:
|
||||||
<!-- vale write-good.Passive = NO -->
|
<!-- vale write-good.Passive = NO -->
|
||||||
|
|
||||||
When OTEL is enabled, **hive-c0re itself** also exports each agent
|
When OTEL is enabled, **hive-c0re itself** also exports each agent
|
||||||
container's resource load — the same cgroup gauges shown on the dashboard
|
container's resource load — the same cgroup gauges —
|
||||||
LOAD tab — to this hive's own collector, exactly like an agent does and with
|
to this hive's own collector, exactly like an agent does and with
|
||||||
no separate toggle. These come from the host, not the in-container Claude SDK,
|
no separate toggle. These come from the host, not the in-container Claude SDK,
|
||||||
so they cover containers even when their agent is idle.
|
so they cover containers even when their agent is idle.
|
||||||
|
|
||||||
|
|
@ -312,7 +312,7 @@ exists, so off-the-shelf OTEL/Grafana container dashboards work. Resource
|
||||||
| `hyperhive.container.memory.limit` | `By` | gauge | `memory.max` (custom — semconv has no `.limit` metric; omitted when unlimited) |
|
| `hyperhive.container.memory.limit` | `By` | gauge | `memory.max` (custom — semconv has no `.limit` metric; omitted when unlimited) |
|
||||||
| `hyperhive.container.memory.peak` | `By` | gauge | `memory.peak` (custom — no semconv metric; omitted if unavailable) |
|
| `hyperhive.container.memory.peak` | `By` | gauge | `memory.peak` (custom — no semconv metric; omitted if unavailable) |
|
||||||
| `hyperhive.container.storage.usage` | `By` | gauge | `state/` + `work/` + writable rootfs, measured by `hive-priv` every 30 min (custom — semconv only has `disk.io`; omitted while unknown: not measured yet, or the last measurement failed) |
|
| `hyperhive.container.storage.usage` | `By` | gauge | `state/` + `work/` + writable rootfs, measured by `hive-priv` every 30 min (custom — semconv only has `disk.io`; omitted while unknown: not measured yet, or the last measurement failed) |
|
||||||
| `hyperhive.container.cpu.percent` | `%` | gauge | host-normalised percent (custom — the value the dashboard LOAD tab shows, no `rate()` needed) |
|
| `hyperhive.container.cpu.percent` | `%` | gauge | host-normalised percent (custom — no `rate()` needed) |
|
||||||
|
|
||||||
The `hyperhive.`-prefixed metrics have no semconv equivalent (memory
|
The `hyperhive.`-prefixed metrics have no semconv equivalent (memory
|
||||||
limit + peak, on-disk footprint, and an instantaneous cpu percent kept
|
limit + peak, on-disk footprint, and an instantaneous cpu percent kept
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,7 @@
|
||||||
//! absent field all fall back to the hive-wide
|
//! absent field all fall back to the hive-wide
|
||||||
//! `services.hyperhive.agentCpuQuota` / `agentMemoryMax`. So a swarm cap on
|
//! `services.hyperhive.agentCpuQuota` / `agentMemoryMax`. So a swarm cap on
|
||||||
//! memory alone leaves CPU on the hive default — see
|
//! memory alone leaves CPU on the hive default — see
|
||||||
//! [`effective_from_with_swarm`].
|
//! [`effective_from`].
|
||||||
//!
|
//!
|
||||||
//! Read path: `lifecycle::host_config::write_dropins`, on every spawn
|
//! Read path: `lifecycle::host_config::write_dropins`, on every spawn
|
||||||
//! and every rebuild.
|
//! and every rebuild.
|
||||||
|
|
|
||||||
|
|
@ -404,9 +404,9 @@ async fn cmd_serve(
|
||||||
// ~hundreds of KB, and the /stats + hive-stats views read it
|
// ~hundreds of KB, and the /stats + hive-stats views read it
|
||||||
// directly; pruning it would just lose trend history for no gain.)
|
// directly; pruning it would just lose trend history for no gain.)
|
||||||
// Slow per-container disk sampler: hive-priv measures each agent's state/
|
// Slow per-container disk sampler: hive-priv measures each agent's state/
|
||||||
// + work/ + writable rootfs every 30 min, cached so the 5s container-load
|
// + work/ + writable rootfs every 30 min, cached so a sample only reads
|
||||||
// poll stays cheap cgroup-only reads. Feeds `disk_bytes` on the LOAD
|
// cgroup files. Feeds `disk_bytes` to the OTEL exporter. See
|
||||||
// tab. See container_stats::disk_sampler_loop.
|
// container_stats::disk_sampler_loop.
|
||||||
crate::container_stats::spawn_disk_sampler();
|
crate::container_stats::spawn_disk_sampler();
|
||||||
// Per-agent container-resource OTEL export: rides the same
|
// Per-agent container-resource OTEL export: rides the same
|
||||||
// cgroup gauges out to the configured OTLP endpoint, reusing the hive
|
// cgroup gauges out to the configured OTLP endpoint, reusing the hive
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
//! Live per-agent-container resource load (CPU + memory, plus the slow
|
//! Live per-agent-container resource load (CPU + memory, plus the slow
|
||||||
//! disk sample from hive-priv in [`disk_sampler_loop`]) for the
|
//! disk sample from hive-priv in [`disk_sampler_loop`]) for the OTEL
|
||||||
//! dashboard. Reads cgroup v2 stat files for each running agent
|
//! exporter. Reads cgroup v2 stat files for each running agent
|
||||||
//! machine directly — same privsep-clean posture as the turn-stats
|
//! machine directly — same privsep-clean posture as the turn-stats
|
||||||
//! sqlite reads (`cpu.stat` / `memory.*` are world-readable; no
|
//! sqlite reads (`cpu.stat` / `memory.*` are world-readable; no
|
||||||
//! `hive-priv` needed). Runs host-side in hive-c0re, where
|
//! `hive-priv` needed). Runs host-side in hive-c0re, where
|
||||||
|
|
@ -31,8 +31,8 @@ use utoipa::ToSchema;
|
||||||
|
|
||||||
use crate::coordinator::Coordinator;
|
use crate::coordinator::Coordinator;
|
||||||
|
|
||||||
/// Sampling interval for the two CPU reads. Short enough to keep the
|
/// Sampling interval for the two CPU reads. Short enough to keep a sample
|
||||||
/// endpoint snappy, long enough that the delta isn't dominated by read
|
/// quick, long enough that the delta isn't dominated by read
|
||||||
/// jitter.
|
/// jitter.
|
||||||
const CPU_SAMPLE: Duration = Duration::from_millis(200);
|
const CPU_SAMPLE: Duration = Duration::from_millis(200);
|
||||||
|
|
||||||
|
|
@ -72,8 +72,8 @@ pub struct ContainerResource {
|
||||||
const DISK_SAMPLE_INTERVAL: Duration = Duration::from_mins(30);
|
const DISK_SAMPLE_INTERVAL: Duration = Duration::from_mins(30);
|
||||||
|
|
||||||
/// Last per-agent disk measurement, keyed by agent name; `None` is a failed
|
/// Last per-agent disk measurement, keyed by agent name; `None` is a failed
|
||||||
/// one. Written by [`disk_sampler_loop`], read by [`gather`] so the hot
|
/// one. Written by [`disk_sampler_loop`], read by [`gather`], so a sample
|
||||||
/// poll never measures.
|
/// never measures.
|
||||||
fn disk_cache() -> &'static RwLock<HashMap<String, Option<u64>>> {
|
fn disk_cache() -> &'static RwLock<HashMap<String, Option<u64>>> {
|
||||||
static CACHE: OnceLock<RwLock<HashMap<String, Option<u64>>>> = OnceLock::new();
|
static CACHE: OnceLock<RwLock<HashMap<String, Option<u64>>>> = OnceLock::new();
|
||||||
CACHE.get_or_init(|| RwLock::new(HashMap::new()))
|
CACHE.get_or_init(|| RwLock::new(HashMap::new()))
|
||||||
|
|
@ -100,7 +100,7 @@ fn disk_reading(name: &hive_types::Ident, measured: anyhow::Result<u64>) -> Opti
|
||||||
|
|
||||||
/// Background loop: every [`DISK_SAMPLE_INTERVAL`], remeasure disk usage
|
/// Background loop: every [`DISK_SAMPLE_INTERVAL`], remeasure disk usage
|
||||||
/// for every kept-state agent and refresh [`disk_cache`]. The first pass
|
/// for every kept-state agent and refresh [`disk_cache`]. The first pass
|
||||||
/// runs immediately so the dashboard shows disk shortly after boot.
|
/// runs immediately so the exporter has disk shortly after boot.
|
||||||
/// Runs forever; spawn once at hive-c0re startup via [`spawn_disk_sampler`].
|
/// Runs forever; spawn once at hive-c0re startup via [`spawn_disk_sampler`].
|
||||||
pub async fn disk_sampler_loop() {
|
pub async fn disk_sampler_loop() {
|
||||||
loop {
|
loop {
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue