container stats: sample the ci runner's container
This commit is contained in:
atlas
parent
530137a7bb
commit
817c222951
5 changed files with 125 additions and 42 deletions
|
|
@ -338,7 +338,11 @@ Emitted via the OpenTelemetry Rust SDK, using the
|
|||
metric names + the standard `container.name` attribute where a spec metric
|
||||
exists, so off-the-shelf OTEL/Grafana container dashboards work. Resource
|
||||
`service.name = hyperhive-c0re`; hive-c0re tags each data point `container.name`
|
||||
(= the `h-<agent>` machine) and the hive `agent` label:
|
||||
(the `h-<agent>` machine) and the hive `agent` label. On a host that runs the
|
||||
CI runner, the same gauges also cover its `swarm-ci-runner` container, with
|
||||
`container.name = swarm-ci-runner` and no `agent` label; the agents dashboard
|
||||
filters those series out and the host dashboard's top-10 panels group by
|
||||
`container.name`. Its storage usage isn't measured, so that series is absent.
|
||||
|
||||
| Metric | Unit | Kind | Source |
|
||||
| --------------------------------------- | ---- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
//! Live per-agent-container resource load (CPU + memory, plus the slow
|
||||
//! Live per-container resource load (agents and the CI runner) (CPU + memory, plus the slow
|
||||
//! disk sample from hive-priv in [`disk_sampler_loop`]) for the OTEL
|
||||
//! exporter. Reads cgroup v2 stat files for each running agent
|
||||
//! machine directly — same privsep-clean posture as the turn-stats
|
||||
|
|
@ -8,7 +8,7 @@
|
|||
//! nixos-container runs `systemd-nspawn --keep-unit` (with
|
||||
//! `Slice = "machine.slice"`), each container's cgroup is its launching
|
||||
//! service unit `container@<machine>.service` — not a machined
|
||||
//! `machine-<name>.scope`. See [`scope_dir`].
|
||||
//! `machine-<name>.scope`. See [`scope_dir_in`].
|
||||
//!
|
||||
//! No network: agents share the host network namespace
|
||||
//! (`privateNetwork = false`), so there is no per-container net
|
||||
|
|
@ -38,10 +38,19 @@ const CPU_SAMPLE: Duration = Duration::from_millis(200);
|
|||
|
||||
const MACHINE_SLICE: &str = "/sys/fs/cgroup/machine.slice";
|
||||
|
||||
/// Machine name of the CI runner's container (`containers.swarm-ci-runner`
|
||||
/// in `nix/host-modules/swarm-ci-runner.nix`).
|
||||
pub const CI_MACHINE: &str = "swarm-ci-runner";
|
||||
|
||||
#[derive(Debug, Serialize, ToSchema)]
|
||||
pub struct ContainerResource {
|
||||
/// Agent name (without the `h-` machine prefix).
|
||||
/// Machine name, the OTEL `container.name`: `h-<agent>` for an agent,
|
||||
/// [`CI_MACHINE`] for the CI runner.
|
||||
pub name: String,
|
||||
/// The agent this container belongs to; `None` for the CI runner, whose
|
||||
/// series therefore carry no `agent` label and stay out of the agents
|
||||
/// dashboard.
|
||||
pub agent: Option<String>,
|
||||
/// Host-normalised CPU usage over the sample interval, as a
|
||||
/// percentage of total host CPU (0..100 across all cores).
|
||||
pub cpu_pct: f64,
|
||||
|
|
@ -134,8 +143,9 @@ pub fn spawn_disk_sampler() {
|
|||
/// used verbatim — no `\x2d` escaping (that only applies when a string is
|
||||
/// converted *into* a scope/slice unit name, not to an already-formed
|
||||
/// instance unit; the journal shows the literal `container@h-<agent>.service`).
|
||||
fn scope_dir(machine: &str) -> PathBuf {
|
||||
PathBuf::from(MACHINE_SLICE).join(format!("container@{machine}.service"))
|
||||
/// `slice` is the `machine.slice` root, a parameter so tests can use a tempdir.
|
||||
fn scope_dir_in(slice: &std::path::Path, machine: &str) -> PathBuf {
|
||||
slice.join(format!("container@{machine}.service"))
|
||||
}
|
||||
|
||||
/// Read a single unsigned integer from a one-line cgroup file.
|
||||
|
|
@ -170,6 +180,26 @@ fn read_usage_usec(dir: &std::path::Path) -> Option<u64> {
|
|||
None
|
||||
}
|
||||
|
||||
/// A running container to sample: machine name, owning agent (`None` for the CI
|
||||
/// runner) and cgroup directory.
|
||||
type Candidate = (String, Option<String>, PathBuf);
|
||||
|
||||
/// The containers under `slice` that currently have a cgroup (= running): one
|
||||
/// per agent in `agents` (machine `h-<agent>`), plus the CI runner's container
|
||||
/// when this host runs one.
|
||||
fn running_candidates(slice: &std::path::Path, agents: &[String]) -> Vec<Candidate> {
|
||||
let running = |machine: &str| {
|
||||
let dir = scope_dir_in(slice, machine);
|
||||
dir.join("cpu.stat").exists().then_some(dir)
|
||||
};
|
||||
let agent_rows = agents.iter().filter_map(|agent| {
|
||||
let machine = format!("h-{agent}");
|
||||
running(&machine).map(|dir| (machine, Some(agent.clone()), dir))
|
||||
});
|
||||
let ci_row = running(CI_MACHINE).map(|dir| (CI_MACHINE.to_owned(), None, dir));
|
||||
agent_rows.chain(ci_row).collect()
|
||||
}
|
||||
|
||||
/// Total online host CPUs, for normalising CPU% to "% of total host".
|
||||
/// Reads `/sys/devices/system/cpu/online` (a comma-separated list of
|
||||
/// ranges, e.g. `0-19`) rather than `available_parallelism()`, which
|
||||
|
|
@ -201,25 +231,24 @@ fn host_nproc() -> usize {
|
|||
.unwrap_or(1)
|
||||
}
|
||||
|
||||
/// Sample CPU + memory for every running agent container. Best-effort:
|
||||
/// agents whose scope dir is absent (not running) or unreadable are
|
||||
/// skipped, never fatal. Sorted by name for a stable display order.
|
||||
/// Sample CPU + memory for every running agent container and for the CI
|
||||
/// runner's container where this host has one. Best-effort: containers whose
|
||||
/// scope dir is absent (not running) or unreadable are skipped, never fatal. Sorted by name for a stable display order.
|
||||
pub async fn gather() -> Vec<ContainerResource> {
|
||||
// Machines that currently have a cgroup scope (= running). Agent
|
||||
// machine name is `h-<state-dir name>`.
|
||||
let candidates: Vec<(String, PathBuf)> = Coordinator::kept_state_names()
|
||||
.into_iter()
|
||||
.filter_map(|name| {
|
||||
let dir = scope_dir(&format!("h-{name}"));
|
||||
dir.join("cpu.stat")
|
||||
.exists()
|
||||
.then_some((name.into_string(), dir))
|
||||
})
|
||||
.collect();
|
||||
let candidates = running_candidates(
|
||||
std::path::Path::new(MACHINE_SLICE),
|
||||
&Coordinator::kept_state_names()
|
||||
.into_iter()
|
||||
.map(hive_types::Ident::into_string)
|
||||
.collect::<Vec<_>>(),
|
||||
);
|
||||
|
||||
// First CPU sample for all, then one shared sleep, then the second
|
||||
// — so N agents cost one interval, not N.
|
||||
let t0: Vec<Option<u64>> = candidates.iter().map(|(_, d)| read_usage_usec(d)).collect();
|
||||
let t0: Vec<Option<u64>> = candidates
|
||||
.iter()
|
||||
.map(|(_, _, d)| read_usage_usec(d))
|
||||
.collect();
|
||||
sleep(CPU_SAMPLE).await;
|
||||
|
||||
let nproc = f64::from(u32::try_from(host_nproc()).unwrap_or(u32::MAX));
|
||||
|
|
@ -231,7 +260,7 @@ pub async fn gather() -> Vec<ContainerResource> {
|
|||
let disk = disk_cache().read().map(|c| c.clone()).unwrap_or_default();
|
||||
|
||||
let mut out: Vec<ContainerResource> = Vec::with_capacity(candidates.len());
|
||||
for (i, (name, dir)) in candidates.iter().enumerate() {
|
||||
for (i, (machine, agent, dir)) in candidates.iter().enumerate() {
|
||||
// The second cumulative read — also exposed raw as `cpu_time_usec`
|
||||
// for the OTEL `container.cpu.time` counter (which wants absolute
|
||||
// cumulative CPU time, not the sampled percent).
|
||||
|
|
@ -244,13 +273,14 @@ pub async fn gather() -> Vec<ContainerResource> {
|
|||
_ => 0.0,
|
||||
};
|
||||
out.push(ContainerResource {
|
||||
name: name.clone(),
|
||||
name: machine.clone(),
|
||||
agent: agent.clone(),
|
||||
cpu_pct,
|
||||
cpu_time_usec: usage_now,
|
||||
mem_current_bytes: read_u64(&dir.join("memory.current")).unwrap_or(0),
|
||||
mem_peak_bytes: read_u64(&dir.join("memory.peak")),
|
||||
mem_max_bytes: read_mem_max(&dir.join("memory.max")),
|
||||
disk_bytes: disk.get(name).copied().flatten(),
|
||||
disk_bytes: agent.as_ref().and_then(|a| disk.get(a).copied().flatten()),
|
||||
});
|
||||
}
|
||||
out
|
||||
|
|
@ -261,21 +291,53 @@ mod tests {
|
|||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn scope_dir_is_the_keep_unit_service_under_machine_slice() {
|
||||
fn scope_dir_in_is_the_keep_unit_service_under_machine_slice() {
|
||||
// nixos-container `--keep-unit` keeps the cgroup on the service
|
||||
// unit `container@<machine>.service` (literal name, no `\x2d`),
|
||||
// under machine.slice — NOT a machined `machine-<name>.scope`.
|
||||
assert_eq!(
|
||||
scope_dir("h-atlas"),
|
||||
scope_dir_in(std::path::Path::new(MACHINE_SLICE), "h-atlas"),
|
||||
PathBuf::from("/sys/fs/cgroup/machine.slice/container@h-atlas.service")
|
||||
);
|
||||
// Underscores in agent names are likewise verbatim.
|
||||
assert_eq!(
|
||||
scope_dir("h-foo_bar"),
|
||||
scope_dir_in(std::path::Path::new(MACHINE_SLICE), "h-foo_bar"),
|
||||
PathBuf::from("/sys/fs/cgroup/machine.slice/container@h-foo_bar.service")
|
||||
);
|
||||
}
|
||||
|
||||
fn touch_running(slice: &std::path::Path, machine: &str) {
|
||||
let dir = scope_dir_in(slice, machine);
|
||||
std::fs::create_dir_all(&dir).expect("mkdir");
|
||||
std::fs::write(dir.join("cpu.stat"), "usage_usec 1\n").expect("write");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_ci_container_is_sampled_when_present_and_skipped_when_absent() {
|
||||
let slice = tempfile::tempdir().expect("tempdir");
|
||||
let agents = vec!["atlas".to_owned()];
|
||||
touch_running(slice.path(), "h-atlas");
|
||||
|
||||
let without = running_candidates(slice.path(), &agents);
|
||||
assert_eq!(without.len(), 1, "absent CI container is skipped");
|
||||
assert_eq!(without[0].0, "h-atlas");
|
||||
assert_eq!(without[0].1.as_deref(), Some("atlas"));
|
||||
|
||||
touch_running(slice.path(), CI_MACHINE);
|
||||
let with = running_candidates(slice.path(), &agents);
|
||||
assert_eq!(with.len(), 2);
|
||||
let ci = with.iter().find(|c| c.0 == CI_MACHINE).expect("ci row");
|
||||
assert_eq!(ci.1, None, "the CI container belongs to no agent");
|
||||
assert_eq!(ci.2, scope_dir_in(slice.path(), CI_MACHINE));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_ci_container_is_sampled_with_no_agents() {
|
||||
let slice = tempfile::tempdir().expect("tempdir");
|
||||
touch_running(slice.path(), CI_MACHINE);
|
||||
assert_eq!(running_candidates(slice.path(), &[]).len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_disk_measurement_is_unknown_not_zero() {
|
||||
let name = hive_types::Ident::parse("iris").expect("valid ident");
|
||||
|
|
|
|||
|
|
@ -423,13 +423,14 @@ fn warning_counts(raised: &[crate::host_stats::ServerWarning]) -> Vec<(&'static
|
|||
}
|
||||
|
||||
/// Per-data-point attributes: the spec `container.name` plus the hive `agent`
|
||||
/// label. (`hive` / `swarm` are constant across this c0re, so they live on the
|
||||
/// label, which the CI runner's container does not carry. (`hive` / `swarm` are constant across this c0re, so they live on the
|
||||
/// resource.)
|
||||
fn attrs(c: &ContainerResource) -> Vec<KeyValue> {
|
||||
vec![
|
||||
KeyValue::new("container.name", format!("h-{}", c.name)),
|
||||
KeyValue::new("agent", c.name.clone()),
|
||||
]
|
||||
let mut kvs = vec![KeyValue::new("container.name", c.name.clone())];
|
||||
if let Some(agent) = &c.agent {
|
||||
kvs.push(KeyValue::new("agent", agent.clone()));
|
||||
}
|
||||
kvs
|
||||
}
|
||||
|
||||
/// Resource: `service.name = hyperhive-c0re`, the host architecture, the
|
||||
|
|
@ -550,7 +551,8 @@ mod tests {
|
|||
// Control: the same predicate, on the per-container attributes, which
|
||||
// are supposed to carry exactly that key.
|
||||
let sample = ContainerResource {
|
||||
name: "atlas".to_owned(),
|
||||
name: "h-atlas".to_owned(),
|
||||
agent: Some("atlas".to_owned()),
|
||||
cpu_pct: 0.0,
|
||||
cpu_time_usec: None,
|
||||
mem_current_bytes: 0,
|
||||
|
|
@ -563,6 +565,21 @@ mod tests {
|
|||
"control failed: the predicate cannot find an `agent` key that IS \
|
||||
present, so the assertion above proves nothing"
|
||||
);
|
||||
|
||||
let ci = ContainerResource {
|
||||
name: container_stats::CI_MACHINE.to_owned(),
|
||||
agent: None,
|
||||
..sample
|
||||
};
|
||||
assert!(
|
||||
!has_agent(&attrs(&ci)),
|
||||
"the CI container carries no agent label"
|
||||
);
|
||||
assert!(
|
||||
attrs(&ci)
|
||||
.iter()
|
||||
.any(|kv| kv.key.as_str() == "container.name")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -146,7 +146,7 @@
|
|||
"name": "@datasourceUid@"
|
||||
},
|
||||
"spec": {
|
||||
"expr": "count(count by (agent) (last_over_time({__name__=\"hyperhive.container.cpu.percent\", hive=~\"$hive\", agent=~\"$agent\"}[$__range])))",
|
||||
"expr": "count(count by (agent) (last_over_time({__name__=\"hyperhive.container.cpu.percent\", hive=~\"$hive\", agent=~\"$agent\", agent!=\"\"}[$__range])))",
|
||||
"instant": true,
|
||||
"legendFormat": "agents"
|
||||
}
|
||||
|
|
@ -289,7 +289,7 @@
|
|||
"name": "@datasourceUid@"
|
||||
},
|
||||
"spec": {
|
||||
"expr": "sum by (agent) ({__name__=\"hyperhive.container.cpu.percent\", hive=~\"$hive\", agent=~\"$agent\"})",
|
||||
"expr": "sum by (agent) ({__name__=\"hyperhive.container.cpu.percent\", hive=~\"$hive\", agent=~\"$agent\", agent!=\"\"})",
|
||||
"legendFormat": "{{agent}}"
|
||||
}
|
||||
},
|
||||
|
|
@ -429,7 +429,7 @@
|
|||
"name": "@datasourceUid@"
|
||||
},
|
||||
"spec": {
|
||||
"expr": "sum by (agent) ({__name__=\"container.memory.usage\", hive=~\"$hive\", agent=~\"$agent\"})",
|
||||
"expr": "sum by (agent) ({__name__=\"container.memory.usage\", hive=~\"$hive\", agent=~\"$agent\", agent!=\"\"})",
|
||||
"legendFormat": "{{agent}}"
|
||||
}
|
||||
},
|
||||
|
|
@ -644,7 +644,7 @@
|
|||
"name": "@datasourceUid@"
|
||||
},
|
||||
"spec": {
|
||||
"expr": "sort_desc(sum by (agent) ({__name__=\"hyperhive.container.storage.usage\", hive=~\"$hive\", agent=~\"$agent\"}))",
|
||||
"expr": "sort_desc(sum by (agent) ({__name__=\"hyperhive.container.storage.usage\", hive=~\"$hive\", agent=~\"$agent\", agent!=\"\"}))",
|
||||
"instant": true,
|
||||
"legendFormat": "{{agent}}"
|
||||
}
|
||||
|
|
@ -3721,7 +3721,7 @@
|
|||
"name": "@datasourceUid@"
|
||||
},
|
||||
"spec": {
|
||||
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.agent.swarm_queue.connected\", hive=~\"$hive\", agent=~\"$agent\"}[5m])) or (max by (agent) (last_over_time({__name__=\"hyperhive.container.cpu.percent\", hive=~\"$hive\", agent=~\"$agent\"}[5m])) * 0 - 1)",
|
||||
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.agent.swarm_queue.connected\", hive=~\"$hive\", agent=~\"$agent\", agent!=\"\"}[5m])) or (max by (agent) (last_over_time({__name__=\"hyperhive.container.cpu.percent\", hive=~\"$hive\", agent=~\"$agent\", agent!=\"\"}[5m])) * 0 - 1)",
|
||||
"legendFormat": "{{agent}}"
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -909,8 +909,8 @@
|
|||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "topk(10, sum by (agent) ({__name__=\"container.memory.usage\", hive=~\"$hive\", \"host.name\"=~\"$host\"}))",
|
||||
"legendFormat": "{{agent}}"
|
||||
"expr": "topk(10, sum by (\"container.name\") ({__name__=\"container.memory.usage\", hive=~\"$hive\", \"host.name\"=~\"$host\"}))",
|
||||
"legendFormat": "{{container.name}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
|
|
@ -965,8 +965,8 @@
|
|||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "topk(10, sum by (agent) (rate({__name__=\"container.cpu.time\", hive=~\"$hive\", \"host.name\"=~\"$host\"}[5m])))",
|
||||
"legendFormat": "{{agent}}"
|
||||
"expr": "topk(10, sum by (\"container.name\") (rate({__name__=\"container.cpu.time\", hive=~\"$hive\", \"host.name\"=~\"$host\"}[5m])))",
|
||||
"legendFormat": "{{container.name}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
|
|
|
|||
Loading…
Reference in a new issue