container stats: sample the ci runner's container

docs/4-implementation/scheduler/observability.md

@@ -338,7 +338,11 @@ Emitted via the OpenTelemetry Rust SDK, using the
 metric names + the standard `container.name` attribute where a spec metric
 exists, so off-the-shelf OTEL/Grafana container dashboards work. Resource
 `service.name = hyperhive-c0re`; hive-c0re tags each data point `container.name`
-(= the `h-<agent>` machine) and the hive `agent` label:
+(the `h-<agent>` machine) and the hive `agent` label. On a host that runs the
+CI runner, the same gauges also cover its `swarm-ci-runner` container, with
+`container.name = swarm-ci-runner` and no `agent` label; the agents dashboard
+filters those series out and the host dashboard's top-10 panels group by
+`container.name`. Its storage usage isn't measured, so that series is absent.
 
 | Metric                                  | Unit | Kind    | Source                                                                                                                                                                                                                                                                                                                                                        |
 | --------------------------------------- | ---- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |

hive-c0re/src/stats/container_stats.rs

@@ -1,4 +1,4 @@
-//! Live per-agent-container resource load (CPU + memory, plus the slow
+//! Live per-container resource load (agents and the CI runner) (CPU + memory, plus the slow
 //! disk sample from hive-priv in [`disk_sampler_loop`]) for the OTEL
 //! exporter. Reads cgroup v2 stat files for each running agent
 //! machine directly — same privsep-clean posture as the turn-stats
@@ -8,7 +8,7 @@
 //! nixos-container runs `systemd-nspawn --keep-unit` (with
 //! `Slice = "machine.slice"`), each container's cgroup is its launching
 //! service unit `container@<machine>.service` — not a machined
-//! `machine-<name>.scope`. See [`scope_dir`].
+//! `machine-<name>.scope`. See [`scope_dir_in`].
 //!
 //! No network: agents share the host network namespace
 //! (`privateNetwork = false`), so there is no per-container net
@@ -38,10 +38,19 @@ const CPU_SAMPLE: Duration = Duration::from_millis(200);
 
 const MACHINE_SLICE: &str = "/sys/fs/cgroup/machine.slice";
 
+/// Machine name of the CI runner's container (`containers.swarm-ci-runner`
+/// in `nix/host-modules/swarm-ci-runner.nix`).
+pub const CI_MACHINE: &str = "swarm-ci-runner";
+
 #[derive(Debug, Serialize, ToSchema)]
 pub struct ContainerResource {
-    /// Agent name (without the `h-` machine prefix).
+    /// Machine name, the OTEL `container.name`: `h-<agent>` for an agent,
+    /// [`CI_MACHINE`] for the CI runner.
     pub name: String,
+    /// The agent this container belongs to; `None` for the CI runner, whose
+    /// series therefore carry no `agent` label and stay out of the agents
+    /// dashboard.
+    pub agent: Option<String>,
     /// Host-normalised CPU usage over the sample interval, as a
     /// percentage of total host CPU (0..100 across all cores).
     pub cpu_pct: f64,
@@ -134,8 +143,9 @@ pub fn spawn_disk_sampler() {
 /// used verbatim — no `\x2d` escaping (that only applies when a string is
 /// converted *into* a scope/slice unit name, not to an already-formed
 /// instance unit; the journal shows the literal `container@h-<agent>.service`).
-fn scope_dir(machine: &str) -> PathBuf {
+/// `slice` is the `machine.slice` root, a parameter so tests can use a tempdir.
-    PathBuf::from(MACHINE_SLICE).join(format!("container@{machine}.service"))
+fn scope_dir_in(slice: &std::path::Path, machine: &str) -> PathBuf {
+    slice.join(format!("container@{machine}.service"))
 }
 
 /// Read a single unsigned integer from a one-line cgroup file.
@@ -170,6 +180,26 @@ fn read_usage_usec(dir: &std::path::Path) -> Option<u64> {
     None
 }
 
+/// A running container to sample: machine name, owning agent (`None` for the CI
+/// runner) and cgroup directory.
+type Candidate = (String, Option<String>, PathBuf);
+
+/// The containers under `slice` that currently have a cgroup (= running): one
+/// per agent in `agents` (machine `h-<agent>`), plus the CI runner's container
+/// when this host runs one.
+fn running_candidates(slice: &std::path::Path, agents: &[String]) -> Vec<Candidate> {
+    let running = |machine: &str| {
+        let dir = scope_dir_in(slice, machine);
+        dir.join("cpu.stat").exists().then_some(dir)
+    };
+    let agent_rows = agents.iter().filter_map(|agent| {
+        let machine = format!("h-{agent}");
+        running(&machine).map(|dir| (machine, Some(agent.clone()), dir))
+    });
+    let ci_row = running(CI_MACHINE).map(|dir| (CI_MACHINE.to_owned(), None, dir));
+    agent_rows.chain(ci_row).collect()
+}
+
 /// Total online host CPUs, for normalising CPU% to "% of total host".
 /// Reads `/sys/devices/system/cpu/online` (a comma-separated list of
 /// ranges, e.g. `0-19`) rather than `available_parallelism()`, which
@@ -201,25 +231,24 @@ fn host_nproc() -> usize {
         .unwrap_or(1)
 }
 
-/// Sample CPU + memory for every running agent container. Best-effort:
+/// Sample CPU + memory for every running agent container and for the CI
-/// agents whose scope dir is absent (not running) or unreadable are
+/// runner's container where this host has one. Best-effort: containers whose
-/// skipped, never fatal. Sorted by name for a stable display order.
+/// scope dir is absent (not running) or unreadable are skipped, never fatal. Sorted by name for a stable display order.
 pub async fn gather() -> Vec<ContainerResource> {
-    // Machines that currently have a cgroup scope (= running). Agent
+    let candidates = running_candidates(
-    // machine name is `h-<state-dir name>`.
+        std::path::Path::new(MACHINE_SLICE),
-    let candidates: Vec<(String, PathBuf)> = Coordinator::kept_state_names()
+        &Coordinator::kept_state_names()
-        .into_iter()
+            .into_iter()
-        .filter_map(|name| {
+            .map(hive_types::Ident::into_string)
-            let dir = scope_dir(&format!("h-{name}"));
+            .collect::<Vec<_>>(),
-            dir.join("cpu.stat")
+    );
-                .exists()
-                .then_some((name.into_string(), dir))
-        })
-        .collect();
 
     // First CPU sample for all, then one shared sleep, then the second
     // — so N agents cost one interval, not N.
-    let t0: Vec<Option<u64>> = candidates.iter().map(|(_, d)| read_usage_usec(d)).collect();
+    let t0: Vec<Option<u64>> = candidates
+        .iter()
+        .map(|(_, _, d)| read_usage_usec(d))
+        .collect();
     sleep(CPU_SAMPLE).await;
 
     let nproc = f64::from(u32::try_from(host_nproc()).unwrap_or(u32::MAX));
@@ -231,7 +260,7 @@ pub async fn gather() -> Vec<ContainerResource> {
     let disk = disk_cache().read().map(|c| c.clone()).unwrap_or_default();
 
     let mut out: Vec<ContainerResource> = Vec::with_capacity(candidates.len());
-    for (i, (name, dir)) in candidates.iter().enumerate() {
+    for (i, (machine, agent, dir)) in candidates.iter().enumerate() {
         // The second cumulative read — also exposed raw as `cpu_time_usec`
         // for the OTEL `container.cpu.time` counter (which wants absolute
         // cumulative CPU time, not the sampled percent).
@@ -244,13 +273,14 @@ pub async fn gather() -> Vec<ContainerResource> {
             _ => 0.0,
         };
         out.push(ContainerResource {
-            name: name.clone(),
+            name: machine.clone(),
+            agent: agent.clone(),
             cpu_pct,
             cpu_time_usec: usage_now,
             mem_current_bytes: read_u64(&dir.join("memory.current")).unwrap_or(0),
             mem_peak_bytes: read_u64(&dir.join("memory.peak")),
             mem_max_bytes: read_mem_max(&dir.join("memory.max")),
-            disk_bytes: disk.get(name).copied().flatten(),
+            disk_bytes: agent.as_ref().and_then(|a| disk.get(a).copied().flatten()),
         });
     }
     out
@@ -261,21 +291,53 @@ mod tests {
     use super::*;
 
     #[test]
-    fn scope_dir_is_the_keep_unit_service_under_machine_slice() {
+    fn scope_dir_in_is_the_keep_unit_service_under_machine_slice() {
         // nixos-container `--keep-unit` keeps the cgroup on the service
         // unit `container@<machine>.service` (literal name, no `\x2d`),
         // under machine.slice — NOT a machined `machine-<name>.scope`.
         assert_eq!(
-            scope_dir("h-atlas"),
+            scope_dir_in(std::path::Path::new(MACHINE_SLICE), "h-atlas"),
             PathBuf::from("/sys/fs/cgroup/machine.slice/container@h-atlas.service")
         );
         // Underscores in agent names are likewise verbatim.
         assert_eq!(
-            scope_dir("h-foo_bar"),
+            scope_dir_in(std::path::Path::new(MACHINE_SLICE), "h-foo_bar"),
             PathBuf::from("/sys/fs/cgroup/machine.slice/container@h-foo_bar.service")
         );
     }
 
+    fn touch_running(slice: &std::path::Path, machine: &str) {
+        let dir = scope_dir_in(slice, machine);
+        std::fs::create_dir_all(&dir).expect("mkdir");
+        std::fs::write(dir.join("cpu.stat"), "usage_usec 1\n").expect("write");
+    }
+
+    #[test]
+    fn the_ci_container_is_sampled_when_present_and_skipped_when_absent() {
+        let slice = tempfile::tempdir().expect("tempdir");
+        let agents = vec!["atlas".to_owned()];
+        touch_running(slice.path(), "h-atlas");
+
+        let without = running_candidates(slice.path(), &agents);
+        assert_eq!(without.len(), 1, "absent CI container is skipped");
+        assert_eq!(without[0].0, "h-atlas");
+        assert_eq!(without[0].1.as_deref(), Some("atlas"));
+
+        touch_running(slice.path(), CI_MACHINE);
+        let with = running_candidates(slice.path(), &agents);
+        assert_eq!(with.len(), 2);
+        let ci = with.iter().find(|c| c.0 == CI_MACHINE).expect("ci row");
+        assert_eq!(ci.1, None, "the CI container belongs to no agent");
+        assert_eq!(ci.2, scope_dir_in(slice.path(), CI_MACHINE));
+    }
+
+    #[test]
+    fn the_ci_container_is_sampled_with_no_agents() {
+        let slice = tempfile::tempdir().expect("tempdir");
+        touch_running(slice.path(), CI_MACHINE);
+        assert_eq!(running_candidates(slice.path(), &[]).len(), 1);
+    }
+
     #[test]
     fn failed_disk_measurement_is_unknown_not_zero() {
         let name = hive_types::Ident::parse("iris").expect("valid ident");

hive-c0re/src/stats/otel_metrics.rs

@@ -423,13 +423,14 @@ fn warning_counts(raised: &[crate::host_stats::ServerWarning]) -> Vec<(&'static
 }
 
 /// Per-data-point attributes: the spec `container.name` plus the hive `agent`
-/// label. (`hive` / `swarm` are constant across this c0re, so they live on the
+/// label, which the CI runner's container does not carry. (`hive` / `swarm` are constant across this c0re, so they live on the
 /// resource.)
 fn attrs(c: &ContainerResource) -> Vec<KeyValue> {
-    vec![
+    let mut kvs = vec![KeyValue::new("container.name", c.name.clone())];
-        KeyValue::new("container.name", format!("h-{}", c.name)),
+    if let Some(agent) = &c.agent {
-        KeyValue::new("agent", c.name.clone()),
+        kvs.push(KeyValue::new("agent", agent.clone()));
-    ]
+    }
+    kvs
 }
 
 /// Resource: `service.name = hyperhive-c0re`, the host architecture, the
@@ -550,7 +551,8 @@ mod tests {
         // Control: the same predicate, on the per-container attributes, which
         // are supposed to carry exactly that key.
         let sample = ContainerResource {
-            name: "atlas".to_owned(),
+            name: "h-atlas".to_owned(),
+            agent: Some("atlas".to_owned()),
             cpu_pct: 0.0,
             cpu_time_usec: None,
             mem_current_bytes: 0,
@@ -563,6 +565,21 @@ mod tests {
             "control failed: the predicate cannot find an `agent` key that IS \
              present, so the assertion above proves nothing"
         );
+
+        let ci = ContainerResource {
+            name: container_stats::CI_MACHINE.to_owned(),
+            agent: None,
+            ..sample
+        };
+        assert!(
+            !has_agent(&attrs(&ci)),
+            "the CI container carries no agent label"
+        );
+        assert!(
+            attrs(&ci)
+                .iter()
+                .any(|kv| kv.key.as_str() == "container.name")
+        );
     }
 
     #[test]

nix/host-modules/swarm-grafana/dashboards/agents.json

@@ -146,7 +146,7 @@
                         "name": "@datasourceUid@"
                       },
                       "spec": {
-                        "expr": "count(count by (agent) (last_over_time({__name__=\"hyperhive.container.cpu.percent\", hive=~\"$hive\", agent=~\"$agent\"}[$__range])))",
+                        "expr": "count(count by (agent) (last_over_time({__name__=\"hyperhive.container.cpu.percent\", hive=~\"$hive\", agent=~\"$agent\", agent!=\"\"}[$__range])))",
                         "instant": true,
                         "legendFormat": "agents"
                       }
@@ -289,7 +289,7 @@
                         "name": "@datasourceUid@"
                       },
                       "spec": {
-                        "expr": "sum by (agent) ({__name__=\"hyperhive.container.cpu.percent\", hive=~\"$hive\", agent=~\"$agent\"})",
+                        "expr": "sum by (agent) ({__name__=\"hyperhive.container.cpu.percent\", hive=~\"$hive\", agent=~\"$agent\", agent!=\"\"})",
                         "legendFormat": "{{agent}}"
                       }
                     },
@@ -429,7 +429,7 @@
                         "name": "@datasourceUid@"
                       },
                       "spec": {
-                        "expr": "sum by (agent) ({__name__=\"container.memory.usage\", hive=~\"$hive\", agent=~\"$agent\"})",
+                        "expr": "sum by (agent) ({__name__=\"container.memory.usage\", hive=~\"$hive\", agent=~\"$agent\", agent!=\"\"})",
                         "legendFormat": "{{agent}}"
                       }
                     },
@@ -644,7 +644,7 @@
                         "name": "@datasourceUid@"
                       },
                       "spec": {
-                        "expr": "sort_desc(sum by (agent) ({__name__=\"hyperhive.container.storage.usage\", hive=~\"$hive\", agent=~\"$agent\"}))",
+                        "expr": "sort_desc(sum by (agent) ({__name__=\"hyperhive.container.storage.usage\", hive=~\"$hive\", agent=~\"$agent\", agent!=\"\"}))",
                         "instant": true,
                         "legendFormat": "{{agent}}"
                       }
@@ -3721,7 +3721,7 @@
                         "name": "@datasourceUid@"
                       },
                       "spec": {
-                        "expr": "max by (agent) (last_over_time({__name__=\"hyperhive.agent.swarm_queue.connected\", hive=~\"$hive\", agent=~\"$agent\"}[5m])) or (max by (agent) (last_over_time({__name__=\"hyperhive.container.cpu.percent\", hive=~\"$hive\", agent=~\"$agent\"}[5m])) * 0 - 1)",
+                        "expr": "max by (agent) (last_over_time({__name__=\"hyperhive.agent.swarm_queue.connected\", hive=~\"$hive\", agent=~\"$agent\", agent!=\"\"}[5m])) or (max by (agent) (last_over_time({__name__=\"hyperhive.container.cpu.percent\", hive=~\"$hive\", agent=~\"$agent\", agent!=\"\"}[5m])) * 0 - 1)",
                         "legendFormat": "{{agent}}"
                       }
                     },

nix/host-modules/swarm-grafana/dashboards/host.json

@@ -909,8 +909,8 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "topk(10, sum by (agent) ({__name__=\"container.memory.usage\", hive=~\"$hive\", \"host.name\"=~\"$host\"}))",
+          "expr": "topk(10, sum by (\"container.name\") ({__name__=\"container.memory.usage\", hive=~\"$hive\", \"host.name\"=~\"$host\"}))",
-          "legendFormat": "{{agent}}"
+          "legendFormat": "{{container.name}}"
         }
       ],
       "fieldConfig": {
@@ -965,8 +965,8 @@
             "type": "prometheus",
             "uid": "@datasourceUid@"
           },
-          "expr": "topk(10, sum by (agent) (rate({__name__=\"container.cpu.time\", hive=~\"$hive\", \"host.name\"=~\"$host\"}[5m])))",
+          "expr": "topk(10, sum by (\"container.name\") (rate({__name__=\"container.cpu.time\", hive=~\"$hive\", \"host.name\"=~\"$host\"}[5m])))",
-          "legendFormat": "{{agent}}"
+          "legendFormat": "{{container.name}}"
         }
       ],
       "fieldConfig": {