diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index cbe8150b..66c7d6c0 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -334,14 +334,15 @@ exists, so off-the-shelf OTEL/Grafana container dashboards work. Resource `service.name = hyperhive-c0re`; hive-c0re tags each data point `container.name` (= the `h-` machine) and the hive `agent` label: -| Metric | Unit | Kind | Source | -| ----------------------------------- | ---- | ------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `container.cpu.time` | `s` | counter | cumulative `cpu.stat` `usage_usec` → seconds | -| `container.memory.usage` | `By` | gauge | `memory.current` | -| `hyperhive.container.memory.limit` | `By` | gauge | `memory.max` (custom — semconv has no `.limit` metric; omitted when unlimited) | -| `hyperhive.container.memory.peak` | `By` | gauge | `memory.peak` (custom — no semconv metric; omitted if unavailable) | -| `hyperhive.container.storage.usage` | `By` | gauge | `state/` + `work/` + writable rootfs, measured by `hive-priv` every 30 min (custom — semconv only has `disk.io`; omitted while unknown: not measured yet, or the last measurement failed) | -| `hyperhive.container.cpu.percent` | `%` | gauge | host-normalised percent (custom — no `rate()` needed) | +| Metric | Unit | Kind | Source | +| --------------------------------------- | ---- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `container.cpu.time` | `s` | counter | cumulative `cpu.stat` `usage_usec` → seconds | +| `container.memory.usage` | `By` | gauge | `memory.current` | +| `hyperhive.container.memory.limit` | `By` | gauge | `memory.max` (custom — semconv has no `.limit` metric; omitted when unlimited) | +| `hyperhive.container.memory.peak` | `By` | gauge | `memory.peak` (custom — no semconv metric; omitted if unavailable) | +| `hyperhive.container.storage.usage` | `By` | gauge | `state/` + `work/` + writable rootfs, measured by `hive-priv` every 30 min (custom — semconv only has `disk.io`; omitted while unknown: not measured yet, or the last measurement failed) | +| `hyperhive.container.cpu.percent` | `%` | gauge | host-normalised percent (custom — no `rate()` needed) | +| `hyperhive.agent.unrestricted_internet` | `1` | gauge | the agent's `unrestricted_internet` flag in the swarm permissions this hive last read: `1` on, `0` off; a flag never set reads `1`. Omitted while the stored permissions file is unreadable | The `hyperhive.`-prefixed metrics have no semconv equivalent (memory limit + peak, on-disk footprint, and an instantaneous cpu percent kept @@ -352,6 +353,13 @@ receiver that accepted the sample. Any other label goes in the hive's +**An absent `hyperhive.agent.unrestricted_internet` series doesn't mean off.** +A hive that predates the metric exports no series at all, and neither does a +hive that can't read its stored permissions. Only a `0` means the flag is off. +Count the agents still on with +`sum({"hyperhive.agent.unrestricted_internet"} == 1)`; dotted names need the +quoted-name selector. + The export cadence is a fixed 60s. Transport is OTLP/HTTP (JSON) to the hive collector's bridge address, with no auth header — that first hop is unauthenticated for every producer on this host, and the upstream diff --git a/hive-c0re/src/agent_config/permissions.rs b/hive-c0re/src/agent_config/permissions.rs index 62a0e4a6..53ee0220 100644 --- a/hive-c0re/src/agent_config/permissions.rs +++ b/hive-c0re/src/agent_config/permissions.rs @@ -47,6 +47,14 @@ pub fn effective(record: Option<&AgentPermissions>) -> Vec { .unwrap_or_else(|| Permission::DEFAULT.to_vec()) } +/// Whether the agent's traffic may leave the host unfiltered under `record`: +/// its flag when set, **on** when the agent has no record or the record never +/// set it. +#[must_use] +pub fn unrestricted_internet(record: Option<&AgentPermissions>) -> bool { + record.is_none_or(swarm_queue_client::agent_permissions::unrestricted) +} + /// Whether `agent` holds `perm` per the file at `path` (normally [`path`]). /// /// # Errors @@ -75,6 +83,19 @@ mod tests { ); } + /// An agent with no record, or a record that never set the flag, reads on. + #[test] + fn unrestricted_internet_reads_absent_as_on() { + assert!(unrestricted_internet(None), "no record"); + assert!(unrestricted_internet(Some(&record("{}"))), "flag unset"); + assert!(unrestricted_internet(Some(&record( + r#"{"unrestricted_internet":true}"# + )))); + assert!(!unrestricted_internet(Some(&record( + r#"{"unrestricted_internet":false}"# + )))); + } + /// No name holds more than the default without an entry: `ruth` with no /// entry holds `DEFAULT`, and only an entry grants her `manage_root_agent`. #[test] diff --git a/hive-c0re/src/stats/otel_metrics.rs b/hive-c0re/src/stats/otel_metrics.rs index 1b099f71..aaa5df2c 100644 --- a/hive-c0re/src/stats/otel_metrics.rs +++ b/hive-c0re/src/stats/otel_metrics.rs @@ -39,6 +39,7 @@ use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider}; use super::container_stats::{self, ContainerResource}; use super::warnings; +use crate::permissions; /// Export cadence, and the snapshot refresher's. const EXPORT_INTERVAL: Duration = Duration::from_mins(1); @@ -223,7 +224,7 @@ fn register_instruments(provider: &SdkMeterProvider, snapshot: Arc u64 { + u64::from(permissions::unrestricted_internet(stored.get(agent))) +} + /// Register the instruments whose subject is **this hive**, not a container in /// it. Each observes with an empty attribute set (`hive` / `swarm` ride the /// resource), so no data point here carries `agent` — see the module doc for @@ -519,6 +545,29 @@ mod tests { ); } + #[test] + fn unrestricted_gauge_reads_absent_as_one() { + let record = |json: &str| serde_json::from_str(json).expect("decodes"); + let stored = permissions::Stored::from([ + ( + "closed".to_owned(), + record(r#"{"unrestricted_internet":false}"#), + ), + ( + "open".to_owned(), + record(r#"{"unrestricted_internet":true}"#), + ), + ( + "legacy".to_owned(), + record(r#"{"permissions":["web_tools"]}"#), + ), + ]); + assert_eq!(unrestricted_value(&stored, "absent"), 1, "no record"); + assert_eq!(unrestricted_value(&stored, "legacy"), 1, "flag unset"); + assert_eq!(unrestricted_value(&stored, "open"), 1); + assert_eq!(unrestricted_value(&stored, "closed"), 0); + } + /// A healthy hive still reports every level, at zero. Reporting only the /// levels currently raised would make a healthy hive and a hive that has /// stopped exporting produce the same thing — no series.