hive-c0re: export the agent's unrestricted internet flag as a gauge
This commit is contained in:
parent
bcb15805d0
commit
24beb167d8
3 changed files with 87 additions and 9 deletions
|
|
@ -334,14 +334,15 @@ exists, so off-the-shelf OTEL/Grafana container dashboards work. Resource
|
||||||
`service.name = hyperhive-c0re`; hive-c0re tags each data point `container.name`
|
`service.name = hyperhive-c0re`; hive-c0re tags each data point `container.name`
|
||||||
(= the `h-<agent>` machine) and the hive `agent` label:
|
(= the `h-<agent>` machine) and the hive `agent` label:
|
||||||
|
|
||||||
| Metric | Unit | Kind | Source |
|
| Metric | Unit | Kind | Source |
|
||||||
| ----------------------------------- | ---- | ------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
| --------------------------------------- | ---- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
| `container.cpu.time` | `s` | counter | cumulative `cpu.stat` `usage_usec` → seconds |
|
| `container.cpu.time` | `s` | counter | cumulative `cpu.stat` `usage_usec` → seconds |
|
||||||
| `container.memory.usage` | `By` | gauge | `memory.current` |
|
| `container.memory.usage` | `By` | gauge | `memory.current` |
|
||||||
| `hyperhive.container.memory.limit` | `By` | gauge | `memory.max` (custom — semconv has no `.limit` metric; omitted when unlimited) |
|
| `hyperhive.container.memory.limit` | `By` | gauge | `memory.max` (custom — semconv has no `.limit` metric; omitted when unlimited) |
|
||||||
| `hyperhive.container.memory.peak` | `By` | gauge | `memory.peak` (custom — no semconv metric; omitted if unavailable) |
|
| `hyperhive.container.memory.peak` | `By` | gauge | `memory.peak` (custom — no semconv metric; omitted if unavailable) |
|
||||||
| `hyperhive.container.storage.usage` | `By` | gauge | `state/` + `work/` + writable rootfs, measured by `hive-priv` every 30 min (custom — semconv only has `disk.io`; omitted while unknown: not measured yet, or the last measurement failed) |
|
| `hyperhive.container.storage.usage` | `By` | gauge | `state/` + `work/` + writable rootfs, measured by `hive-priv` every 30 min (custom — semconv only has `disk.io`; omitted while unknown: not measured yet, or the last measurement failed) |
|
||||||
| `hyperhive.container.cpu.percent` | `%` | gauge | host-normalised percent (custom — no `rate()` needed) |
|
| `hyperhive.container.cpu.percent` | `%` | gauge | host-normalised percent (custom — no `rate()` needed) |
|
||||||
|
| `hyperhive.agent.unrestricted_internet` | `1` | gauge | the agent's `unrestricted_internet` flag in the swarm permissions this hive last read: `1` on, `0` off; a flag never set reads `1`. Omitted while the stored permissions file is unreadable |
|
||||||
|
|
||||||
The `hyperhive.`-prefixed metrics have no semconv equivalent (memory
|
The `hyperhive.`-prefixed metrics have no semconv equivalent (memory
|
||||||
limit + peak, on-disk footprint, and an instantaneous cpu percent kept
|
limit + peak, on-disk footprint, and an instantaneous cpu percent kept
|
||||||
|
|
@ -352,6 +353,13 @@ receiver that accepted the sample. Any other label goes in the hive's
|
||||||
|
|
||||||
<!-- vale write-good.Passive = NO -->
|
<!-- vale write-good.Passive = NO -->
|
||||||
|
|
||||||
|
**An absent `hyperhive.agent.unrestricted_internet` series doesn't mean off.**
|
||||||
|
A hive that predates the metric exports no series at all, and neither does a
|
||||||
|
hive that can't read its stored permissions. Only a `0` means the flag is off.
|
||||||
|
Count the agents still on with
|
||||||
|
`sum({"hyperhive.agent.unrestricted_internet"} == 1)`; dotted names need the
|
||||||
|
quoted-name selector.
|
||||||
|
|
||||||
The export cadence is a fixed 60s. Transport is OTLP/HTTP
|
The export cadence is a fixed 60s. Transport is OTLP/HTTP
|
||||||
(JSON) to the hive collector's bridge address, with no auth header — that
|
(JSON) to the hive collector's bridge address, with no auth header — that
|
||||||
first hop is unauthenticated for every producer on this host, and the upstream
|
first hop is unauthenticated for every producer on this host, and the upstream
|
||||||
|
|
|
||||||
|
|
@ -47,6 +47,14 @@ pub fn effective(record: Option<&AgentPermissions>) -> Vec<Permission> {
|
||||||
.unwrap_or_else(|| Permission::DEFAULT.to_vec())
|
.unwrap_or_else(|| Permission::DEFAULT.to_vec())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether the agent's traffic may leave the host unfiltered under `record`:
|
||||||
|
/// its flag when set, **on** when the agent has no record or the record never
|
||||||
|
/// set it.
|
||||||
|
#[must_use]
|
||||||
|
pub fn unrestricted_internet(record: Option<&AgentPermissions>) -> bool {
|
||||||
|
record.is_none_or(swarm_queue_client::agent_permissions::unrestricted)
|
||||||
|
}
|
||||||
|
|
||||||
/// Whether `agent` holds `perm` per the file at `path` (normally [`path`]).
|
/// Whether `agent` holds `perm` per the file at `path` (normally [`path`]).
|
||||||
///
|
///
|
||||||
/// # Errors
|
/// # Errors
|
||||||
|
|
@ -75,6 +83,19 @@ mod tests {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// An agent with no record, or a record that never set the flag, reads on.
|
||||||
|
#[test]
|
||||||
|
fn unrestricted_internet_reads_absent_as_on() {
|
||||||
|
assert!(unrestricted_internet(None), "no record");
|
||||||
|
assert!(unrestricted_internet(Some(&record("{}"))), "flag unset");
|
||||||
|
assert!(unrestricted_internet(Some(&record(
|
||||||
|
r#"{"unrestricted_internet":true}"#
|
||||||
|
))));
|
||||||
|
assert!(!unrestricted_internet(Some(&record(
|
||||||
|
r#"{"unrestricted_internet":false}"#
|
||||||
|
))));
|
||||||
|
}
|
||||||
|
|
||||||
/// No name holds more than the default without an entry: `ruth` with no
|
/// No name holds more than the default without an entry: `ruth` with no
|
||||||
/// entry holds `DEFAULT`, and only an entry grants her `manage_root_agent`.
|
/// entry holds `DEFAULT`, and only an entry grants her `manage_root_agent`.
|
||||||
#[test]
|
#[test]
|
||||||
|
|
|
||||||
|
|
@ -39,6 +39,7 @@ use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider};
|
||||||
|
|
||||||
use super::container_stats::{self, ContainerResource};
|
use super::container_stats::{self, ContainerResource};
|
||||||
use super::warnings;
|
use super::warnings;
|
||||||
|
use crate::permissions;
|
||||||
|
|
||||||
/// Export cadence, and the snapshot refresher's.
|
/// Export cadence, and the snapshot refresher's.
|
||||||
const EXPORT_INTERVAL: Duration = Duration::from_mins(1);
|
const EXPORT_INTERVAL: Duration = Duration::from_mins(1);
|
||||||
|
|
@ -223,7 +224,7 @@ fn register_instruments(provider: &SdkMeterProvider, snapshot: Arc<Mutex<Vec<Con
|
||||||
})
|
})
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
let snap = snapshot;
|
let snap = snapshot.clone();
|
||||||
meter
|
meter
|
||||||
.f64_observable_gauge("hyperhive.container.cpu.percent")
|
.f64_observable_gauge("hyperhive.container.cpu.percent")
|
||||||
.with_unit("%")
|
.with_unit("%")
|
||||||
|
|
@ -236,9 +237,34 @@ fn register_instruments(provider: &SdkMeterProvider, snapshot: Arc<Mutex<Vec<Con
|
||||||
})
|
})
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
|
// 1 when the agent's traffic may leave the host unfiltered. Read from the
|
||||||
|
// stored swarm permissions at each export; when that file can't be read
|
||||||
|
// the callback reports nothing rather than a guess.
|
||||||
|
let snap = snapshot;
|
||||||
|
meter
|
||||||
|
.u64_observable_gauge("hyperhive.agent.unrestricted_internet")
|
||||||
|
.with_unit("1")
|
||||||
|
.with_callback(move |obs| {
|
||||||
|
let Ok(stored) = permissions::read_at(&permissions::path()) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
if let Ok(g) = snap.lock() {
|
||||||
|
for c in g.iter() {
|
||||||
|
obs.observe(unrestricted_value(&stored, &c.name), &attrs(c));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.build();
|
||||||
|
|
||||||
register_hive_instruments(&meter);
|
register_hive_instruments(&meter);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The gauge value for `agent`: 1 when unrestricted, 0 when not. An agent
|
||||||
|
/// missing from `stored` reads 1.
|
||||||
|
fn unrestricted_value(stored: &permissions::Stored, agent: &str) -> u64 {
|
||||||
|
u64::from(permissions::unrestricted_internet(stored.get(agent)))
|
||||||
|
}
|
||||||
|
|
||||||
/// Register the instruments whose subject is **this hive**, not a container in
|
/// Register the instruments whose subject is **this hive**, not a container in
|
||||||
/// it. Each observes with an empty attribute set (`hive` / `swarm` ride the
|
/// it. Each observes with an empty attribute set (`hive` / `swarm` ride the
|
||||||
/// resource), so no data point here carries `agent` — see the module doc for
|
/// resource), so no data point here carries `agent` — see the module doc for
|
||||||
|
|
@ -519,6 +545,29 @@ mod tests {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unrestricted_gauge_reads_absent_as_one() {
|
||||||
|
let record = |json: &str| serde_json::from_str(json).expect("decodes");
|
||||||
|
let stored = permissions::Stored::from([
|
||||||
|
(
|
||||||
|
"closed".to_owned(),
|
||||||
|
record(r#"{"unrestricted_internet":false}"#),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"open".to_owned(),
|
||||||
|
record(r#"{"unrestricted_internet":true}"#),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"legacy".to_owned(),
|
||||||
|
record(r#"{"permissions":["web_tools"]}"#),
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
assert_eq!(unrestricted_value(&stored, "absent"), 1, "no record");
|
||||||
|
assert_eq!(unrestricted_value(&stored, "legacy"), 1, "flag unset");
|
||||||
|
assert_eq!(unrestricted_value(&stored, "open"), 1);
|
||||||
|
assert_eq!(unrestricted_value(&stored, "closed"), 0);
|
||||||
|
}
|
||||||
|
|
||||||
/// A healthy hive still reports every level, at zero. Reporting only the
|
/// A healthy hive still reports every level, at zero. Reporting only the
|
||||||
/// levels currently raised would make a healthy hive and a hive that has
|
/// levels currently raised would make a healthy hive and a hive that has
|
||||||
/// stopped exporting produce the same thing — no series.
|
/// stopped exporting produce the same thing — no series.
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue