diff --git a/docs/4-implementation/scheduler/observability.md b/docs/4-implementation/scheduler/observability.md index 0c4125d3..f9a6faed 100644 --- a/docs/4-implementation/scheduler/observability.md +++ b/docs/4-implementation/scheduler/observability.md @@ -338,7 +338,11 @@ Emitted via the OpenTelemetry Rust SDK, using the metric names + the standard `container.name` attribute where a spec metric exists, so off-the-shelf OTEL/Grafana container dashboards work. Resource `service.name = hyperhive-c0re`; hive-c0re tags each data point `container.name` -(= the `h-` machine) and the hive `agent` label: +(the `h-` machine) and the hive `agent` label. On a host that runs the +CI runner, the same gauges also cover its `swarm-ci-runner` container, with +`container.name = swarm-ci-runner` and no `agent` label; the agents dashboard +filters those series out and the host dashboard's top-10 panels group by +`container.name`. Its storage usage isn't measured, so that series is absent. | Metric | Unit | Kind | Source | | --------------------------------------- | ---- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | diff --git a/hive-c0re/src/stats/container_stats.rs b/hive-c0re/src/stats/container_stats.rs index 8b8e4827..0cd39c96 100644 --- a/hive-c0re/src/stats/container_stats.rs +++ b/hive-c0re/src/stats/container_stats.rs @@ -1,4 +1,4 @@ -//! Live per-agent-container resource load (CPU + memory, plus the slow +//! Live per-container resource load (agents and the CI runner) (CPU + memory, plus the slow //! disk sample from hive-priv in [`disk_sampler_loop`]) for the OTEL //! exporter. Reads cgroup v2 stat files for each running agent //! machine directly — same privsep-clean posture as the turn-stats @@ -8,7 +8,7 @@ //! nixos-container runs `systemd-nspawn --keep-unit` (with //! `Slice = "machine.slice"`), each container's cgroup is its launching //! service unit `container@.service` — not a machined -//! `machine-.scope`. See [`scope_dir`]. +//! `machine-.scope`. See [`scope_dir_in`]. //! //! No network: agents share the host network namespace //! (`privateNetwork = false`), so there is no per-container net @@ -38,10 +38,19 @@ const CPU_SAMPLE: Duration = Duration::from_millis(200); const MACHINE_SLICE: &str = "/sys/fs/cgroup/machine.slice"; +/// Machine name of the CI runner's container (`containers.swarm-ci-runner` +/// in `nix/host-modules/swarm-ci-runner.nix`). +pub const CI_MACHINE: &str = "swarm-ci-runner"; + #[derive(Debug, Serialize, ToSchema)] pub struct ContainerResource { - /// Agent name (without the `h-` machine prefix). + /// Machine name, the OTEL `container.name`: `h-` for an agent, + /// [`CI_MACHINE`] for the CI runner. pub name: String, + /// The agent this container belongs to; `None` for the CI runner, whose + /// series therefore carry no `agent` label and stay out of the agents + /// dashboard. + pub agent: Option, /// Host-normalised CPU usage over the sample interval, as a /// percentage of total host CPU (0..100 across all cores). pub cpu_pct: f64, @@ -134,8 +143,9 @@ pub fn spawn_disk_sampler() { /// used verbatim — no `\x2d` escaping (that only applies when a string is /// converted *into* a scope/slice unit name, not to an already-formed /// instance unit; the journal shows the literal `container@h-.service`). -fn scope_dir(machine: &str) -> PathBuf { - PathBuf::from(MACHINE_SLICE).join(format!("container@{machine}.service")) +/// `slice` is the `machine.slice` root, a parameter so tests can use a tempdir. +fn scope_dir_in(slice: &std::path::Path, machine: &str) -> PathBuf { + slice.join(format!("container@{machine}.service")) } /// Read a single unsigned integer from a one-line cgroup file. @@ -170,6 +180,26 @@ fn read_usage_usec(dir: &std::path::Path) -> Option { None } +/// A running container to sample: machine name, owning agent (`None` for the CI +/// runner) and cgroup directory. +type Candidate = (String, Option, PathBuf); + +/// The containers under `slice` that currently have a cgroup (= running): one +/// per agent in `agents` (machine `h-`), plus the CI runner's container +/// when this host runs one. +fn running_candidates(slice: &std::path::Path, agents: &[String]) -> Vec { + let running = |machine: &str| { + let dir = scope_dir_in(slice, machine); + dir.join("cpu.stat").exists().then_some(dir) + }; + let agent_rows = agents.iter().filter_map(|agent| { + let machine = format!("h-{agent}"); + running(&machine).map(|dir| (machine, Some(agent.clone()), dir)) + }); + let ci_row = running(CI_MACHINE).map(|dir| (CI_MACHINE.to_owned(), None, dir)); + agent_rows.chain(ci_row).collect() +} + /// Total online host CPUs, for normalising CPU% to "% of total host". /// Reads `/sys/devices/system/cpu/online` (a comma-separated list of /// ranges, e.g. `0-19`) rather than `available_parallelism()`, which @@ -201,25 +231,24 @@ fn host_nproc() -> usize { .unwrap_or(1) } -/// Sample CPU + memory for every running agent container. Best-effort: -/// agents whose scope dir is absent (not running) or unreadable are -/// skipped, never fatal. Sorted by name for a stable display order. +/// Sample CPU + memory for every running agent container and for the CI +/// runner's container where this host has one. Best-effort: containers whose +/// scope dir is absent (not running) or unreadable are skipped, never fatal. Sorted by name for a stable display order. pub async fn gather() -> Vec { - // Machines that currently have a cgroup scope (= running). Agent - // machine name is `h-`. - let candidates: Vec<(String, PathBuf)> = Coordinator::kept_state_names() - .into_iter() - .filter_map(|name| { - let dir = scope_dir(&format!("h-{name}")); - dir.join("cpu.stat") - .exists() - .then_some((name.into_string(), dir)) - }) - .collect(); + let candidates = running_candidates( + std::path::Path::new(MACHINE_SLICE), + &Coordinator::kept_state_names() + .into_iter() + .map(hive_types::Ident::into_string) + .collect::>(), + ); // First CPU sample for all, then one shared sleep, then the second // — so N agents cost one interval, not N. - let t0: Vec> = candidates.iter().map(|(_, d)| read_usage_usec(d)).collect(); + let t0: Vec> = candidates + .iter() + .map(|(_, _, d)| read_usage_usec(d)) + .collect(); sleep(CPU_SAMPLE).await; let nproc = f64::from(u32::try_from(host_nproc()).unwrap_or(u32::MAX)); @@ -231,7 +260,7 @@ pub async fn gather() -> Vec { let disk = disk_cache().read().map(|c| c.clone()).unwrap_or_default(); let mut out: Vec = Vec::with_capacity(candidates.len()); - for (i, (name, dir)) in candidates.iter().enumerate() { + for (i, (machine, agent, dir)) in candidates.iter().enumerate() { // The second cumulative read — also exposed raw as `cpu_time_usec` // for the OTEL `container.cpu.time` counter (which wants absolute // cumulative CPU time, not the sampled percent). @@ -244,13 +273,14 @@ pub async fn gather() -> Vec { _ => 0.0, }; out.push(ContainerResource { - name: name.clone(), + name: machine.clone(), + agent: agent.clone(), cpu_pct, cpu_time_usec: usage_now, mem_current_bytes: read_u64(&dir.join("memory.current")).unwrap_or(0), mem_peak_bytes: read_u64(&dir.join("memory.peak")), mem_max_bytes: read_mem_max(&dir.join("memory.max")), - disk_bytes: disk.get(name).copied().flatten(), + disk_bytes: agent.as_ref().and_then(|a| disk.get(a).copied().flatten()), }); } out @@ -261,21 +291,53 @@ mod tests { use super::*; #[test] - fn scope_dir_is_the_keep_unit_service_under_machine_slice() { + fn scope_dir_in_is_the_keep_unit_service_under_machine_slice() { // nixos-container `--keep-unit` keeps the cgroup on the service // unit `container@.service` (literal name, no `\x2d`), // under machine.slice — NOT a machined `machine-.scope`. assert_eq!( - scope_dir("h-atlas"), + scope_dir_in(std::path::Path::new(MACHINE_SLICE), "h-atlas"), PathBuf::from("/sys/fs/cgroup/machine.slice/container@h-atlas.service") ); // Underscores in agent names are likewise verbatim. assert_eq!( - scope_dir("h-foo_bar"), + scope_dir_in(std::path::Path::new(MACHINE_SLICE), "h-foo_bar"), PathBuf::from("/sys/fs/cgroup/machine.slice/container@h-foo_bar.service") ); } + fn touch_running(slice: &std::path::Path, machine: &str) { + let dir = scope_dir_in(slice, machine); + std::fs::create_dir_all(&dir).expect("mkdir"); + std::fs::write(dir.join("cpu.stat"), "usage_usec 1\n").expect("write"); + } + + #[test] + fn the_ci_container_is_sampled_when_present_and_skipped_when_absent() { + let slice = tempfile::tempdir().expect("tempdir"); + let agents = vec!["atlas".to_owned()]; + touch_running(slice.path(), "h-atlas"); + + let without = running_candidates(slice.path(), &agents); + assert_eq!(without.len(), 1, "absent CI container is skipped"); + assert_eq!(without[0].0, "h-atlas"); + assert_eq!(without[0].1.as_deref(), Some("atlas")); + + touch_running(slice.path(), CI_MACHINE); + let with = running_candidates(slice.path(), &agents); + assert_eq!(with.len(), 2); + let ci = with.iter().find(|c| c.0 == CI_MACHINE).expect("ci row"); + assert_eq!(ci.1, None, "the CI container belongs to no agent"); + assert_eq!(ci.2, scope_dir_in(slice.path(), CI_MACHINE)); + } + + #[test] + fn the_ci_container_is_sampled_with_no_agents() { + let slice = tempfile::tempdir().expect("tempdir"); + touch_running(slice.path(), CI_MACHINE); + assert_eq!(running_candidates(slice.path(), &[]).len(), 1); + } + #[test] fn failed_disk_measurement_is_unknown_not_zero() { let name = hive_types::Ident::parse("iris").expect("valid ident"); diff --git a/hive-c0re/src/stats/otel_metrics.rs b/hive-c0re/src/stats/otel_metrics.rs index cf387708..06d963ad 100644 --- a/hive-c0re/src/stats/otel_metrics.rs +++ b/hive-c0re/src/stats/otel_metrics.rs @@ -423,13 +423,14 @@ fn warning_counts(raised: &[crate::host_stats::ServerWarning]) -> Vec<(&'static } /// Per-data-point attributes: the spec `container.name` plus the hive `agent` -/// label. (`hive` / `swarm` are constant across this c0re, so they live on the +/// label, which the CI runner's container does not carry. (`hive` / `swarm` are constant across this c0re, so they live on the /// resource.) fn attrs(c: &ContainerResource) -> Vec { - vec![ - KeyValue::new("container.name", format!("h-{}", c.name)), - KeyValue::new("agent", c.name.clone()), - ] + let mut kvs = vec![KeyValue::new("container.name", c.name.clone())]; + if let Some(agent) = &c.agent { + kvs.push(KeyValue::new("agent", agent.clone())); + } + kvs } /// Resource: `service.name = hyperhive-c0re`, the host architecture, the @@ -550,7 +551,8 @@ mod tests { // Control: the same predicate, on the per-container attributes, which // are supposed to carry exactly that key. let sample = ContainerResource { - name: "atlas".to_owned(), + name: "h-atlas".to_owned(), + agent: Some("atlas".to_owned()), cpu_pct: 0.0, cpu_time_usec: None, mem_current_bytes: 0, @@ -563,6 +565,21 @@ mod tests { "control failed: the predicate cannot find an `agent` key that IS \ present, so the assertion above proves nothing" ); + + let ci = ContainerResource { + name: container_stats::CI_MACHINE.to_owned(), + agent: None, + ..sample + }; + assert!( + !has_agent(&attrs(&ci)), + "the CI container carries no agent label" + ); + assert!( + attrs(&ci) + .iter() + .any(|kv| kv.key.as_str() == "container.name") + ); } #[test] diff --git a/nix/host-modules/swarm-grafana/dashboards/agents.json b/nix/host-modules/swarm-grafana/dashboards/agents.json index aee67313..b9c22b65 100644 --- a/nix/host-modules/swarm-grafana/dashboards/agents.json +++ b/nix/host-modules/swarm-grafana/dashboards/agents.json @@ -146,7 +146,7 @@ "name": "@datasourceUid@" }, "spec": { - "expr": "count(count by (agent) (last_over_time({__name__=\"hyperhive.container.cpu.percent\", hive=~\"$hive\", agent=~\"$agent\"}[$__range])))", + "expr": "count(count by (agent) (last_over_time({__name__=\"hyperhive.container.cpu.percent\", hive=~\"$hive\", agent=~\"$agent\", agent!=\"\"}[$__range])))", "instant": true, "legendFormat": "agents" } @@ -289,7 +289,7 @@ "name": "@datasourceUid@" }, "spec": { - "expr": "sum by (agent) ({__name__=\"hyperhive.container.cpu.percent\", hive=~\"$hive\", agent=~\"$agent\"})", + "expr": "sum by (agent) ({__name__=\"hyperhive.container.cpu.percent\", hive=~\"$hive\", agent=~\"$agent\", agent!=\"\"})", "legendFormat": "{{agent}}" } }, @@ -429,7 +429,7 @@ "name": "@datasourceUid@" }, "spec": { - "expr": "sum by (agent) ({__name__=\"container.memory.usage\", hive=~\"$hive\", agent=~\"$agent\"})", + "expr": "sum by (agent) ({__name__=\"container.memory.usage\", hive=~\"$hive\", agent=~\"$agent\", agent!=\"\"})", "legendFormat": "{{agent}}" } }, @@ -644,7 +644,7 @@ "name": "@datasourceUid@" }, "spec": { - "expr": "sort_desc(sum by (agent) ({__name__=\"hyperhive.container.storage.usage\", hive=~\"$hive\", agent=~\"$agent\"}))", + "expr": "sort_desc(sum by (agent) ({__name__=\"hyperhive.container.storage.usage\", hive=~\"$hive\", agent=~\"$agent\", agent!=\"\"}))", "instant": true, "legendFormat": "{{agent}}" } @@ -3721,7 +3721,7 @@ "name": "@datasourceUid@" }, "spec": { - "expr": "max by (agent) (last_over_time({__name__=\"hyperhive.agent.swarm_queue.connected\", hive=~\"$hive\", agent=~\"$agent\"}[5m])) or (max by (agent) (last_over_time({__name__=\"hyperhive.container.cpu.percent\", hive=~\"$hive\", agent=~\"$agent\"}[5m])) * 0 - 1)", + "expr": "max by (agent) (last_over_time({__name__=\"hyperhive.agent.swarm_queue.connected\", hive=~\"$hive\", agent=~\"$agent\", agent!=\"\"}[5m])) or (max by (agent) (last_over_time({__name__=\"hyperhive.container.cpu.percent\", hive=~\"$hive\", agent=~\"$agent\", agent!=\"\"}[5m])) * 0 - 1)", "legendFormat": "{{agent}}" } }, diff --git a/nix/host-modules/swarm-grafana/dashboards/host.json b/nix/host-modules/swarm-grafana/dashboards/host.json index 35e86423..6c1c7b09 100644 --- a/nix/host-modules/swarm-grafana/dashboards/host.json +++ b/nix/host-modules/swarm-grafana/dashboards/host.json @@ -909,8 +909,8 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "topk(10, sum by (agent) ({__name__=\"container.memory.usage\", hive=~\"$hive\", \"host.name\"=~\"$host\"}))", - "legendFormat": "{{agent}}" + "expr": "topk(10, sum by (\"container.name\") ({__name__=\"container.memory.usage\", hive=~\"$hive\", \"host.name\"=~\"$host\"}))", + "legendFormat": "{{container.name}}" } ], "fieldConfig": { @@ -965,8 +965,8 @@ "type": "prometheus", "uid": "@datasourceUid@" }, - "expr": "topk(10, sum by (agent) (rate({__name__=\"container.cpu.time\", hive=~\"$hive\", \"host.name\"=~\"$host\"}[5m])))", - "legendFormat": "{{agent}}" + "expr": "topk(10, sum by (\"container.name\") (rate({__name__=\"container.cpu.time\", hive=~\"$hive\", \"host.name\"=~\"$host\"}[5m])))", + "legendFormat": "{{container.name}}" } ], "fieldConfig": {