From d2ea50031e5d76ebefc060b4595955b004d86615 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 5 Oct 2026 13:23:39 +0200 Subject: [PATCH] agents: work/ in the disk-metric docs, holder never sees work/, chown guard comment --- docs/agent-lifecycle/persistence.md | 3 +++ docs/scheduler/observability.md | 16 ++++++++-------- docs/web-ui/dashboard.md | 4 ++-- frontend/packages/dashboard/src/core.js | 4 ++-- hive-c0re/src/lifecycle/host_config.rs | 23 ++++++++++++++++++++++- hive-c0re/src/main.rs | 2 +- hive-c0re/src/stats/container_stats.rs | 2 +- nix/agent-modules/user.nix | 4 +++- 8 files changed, 42 insertions(+), 16 deletions(-) diff --git a/docs/agent-lifecycle/persistence.md b/docs/agent-lifecycle/persistence.md index 58ffff89..d579fdd1 100644 --- a/docs/agent-lifecycle/persistence.md +++ b/docs/agent-lifecycle/persistence.md @@ -462,6 +462,9 @@ sends to the swarm's snapshot store — see [`docs/networking/snapshot-store.md`](../networking/snapshot-store.md) for what a pushed snapshot contains and how the store authenticates a sender. +The subvolume is the whole agent state root, so `work/` is part of every +snapshot and migration of it and can make them much larger. + ## `/var/lib/swarm-controller/` (swarm-controller host only) Only present on the one host running diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index 8711c3c5..83106e6a 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -292,14 +292,14 @@ exists, so off-the-shelf OTEL/Grafana container dashboards work. Resource `service.name = hyperhive-c0re`; hive-c0re tags each data point `container.name` (= the `h-` machine) and the hive `agent` label: -| Metric | Unit | Kind | Source | -| ----------------------------------- | ---- | ------- | ----------------------------------------------------------------------------------------------------------- | -| `container.cpu.time` | `s` | counter | cumulative `cpu.stat` `usage_usec` → seconds | -| `container.memory.usage` | `By` | gauge | `memory.current` | -| `hyperhive.container.memory.limit` | `By` | gauge | `memory.max` (custom — semconv has no `.limit` metric; omitted when unlimited) | -| `hyperhive.container.memory.peak` | `By` | gauge | `memory.peak` (custom — no semconv metric; omitted if unavailable) | -| `hyperhive.container.storage.usage` | `By` | gauge | state dir + writable rootfs (custom — semconv only has `disk.io`; omitted until the slow disk sampler runs) | -| `hyperhive.container.cpu.percent` | `%` | gauge | host-normalised percent (custom — the value the dashboard LOAD tab shows, no `rate()` needed) | +| Metric | Unit | Kind | Source | +| ----------------------------------- | ---- | ------- | -------------------------------------------------------------------------------------------------------------------- | +| `container.cpu.time` | `s` | counter | cumulative `cpu.stat` `usage_usec` → seconds | +| `container.memory.usage` | `By` | gauge | `memory.current` | +| `hyperhive.container.memory.limit` | `By` | gauge | `memory.max` (custom — semconv has no `.limit` metric; omitted when unlimited) | +| `hyperhive.container.memory.peak` | `By` | gauge | `memory.peak` (custom — no semconv metric; omitted if unavailable) | +| `hyperhive.container.storage.usage` | `By` | gauge | `state/` + `work/` + writable rootfs (custom — semconv only has `disk.io`; omitted until the slow disk sampler runs) | +| `hyperhive.container.cpu.percent` | `%` | gauge | host-normalised percent (custom — the value the dashboard LOAD tab shows, no `rate()` needed) | The `hyperhive.`-prefixed metrics have no semconv equivalent (memory limit + peak, on-disk footprint, and an instantaneous cpu percent kept diff --git a/docs/web-ui/dashboard.md b/docs/web-ui/dashboard.md index b24a97bd..654ce9b1 100644 --- a/docs/web-ui/dashboard.md +++ b/docs/web-ui/dashboard.md @@ -157,7 +157,7 @@ scope dir is absent (= not running). Pull-only: `core.js` polls every 5 s **only while the C0NT41N3R L04D sub-tab is active** (CPU needs a fresh sample each refresh), and stops on sub-tab change. Disk size (`disk_bytes`) rides the same row, but a separate ~5 min background `du` sampler feeds -it (state dir + container writable rootfs, shared nix store excluded via +it (`state/` + `work/` + container writable rootfs, shared nix store excluded via `du -x`), so the 5 s poll stays cheap cgroup-only reads; the row carries the last-sampled value (`null` until the first sample). Network isn't tracked per container — see `docs/networking/network.md` for the current @@ -922,7 +922,7 @@ below — some endpoints aren't in it yet. ~200 ms so the call briefly awaits. Skips non-running agents (no scope dir). No network field — agents share the host netns. `disk_bytes` is the last value from a **separate slow sampler** (not this hot path): - a background `du -sxb` of the agent's state dir + container writable + a background `du -sxb` of the agent's `state/` + `work/` + container writable rootfs every ~5 min, `-x` excluding the shared read-only nix store. `null` until the first sample lands. - `POST /api/cancel-reminder/{id}` — hard-delete a pending reminder. diff --git a/frontend/packages/dashboard/src/core.js b/frontend/packages/dashboard/src/core.js index 9aa8b7a1..848626a2 100644 --- a/frontend/packages/dashboard/src/core.js +++ b/frontend/packages/dashboard/src/core.js @@ -285,13 +285,13 @@ function renderContainerLoad(rows) { ? cloadFmtBytes(Number(r.mem_max_bytes)) : "∞"; tr.append(lim); - // Disk: on-disk footprint (state dir + container writable rootfs, shared + // Disk: on-disk footprint (state/ + work/ + container writable rootfs, shared // nix store excluded). Sampled out-of-band every few minutes server-side, // so it's null until the first sample lands — show an em-dash then. const disk = document.createElement("td"); disk.className = "num"; disk.title = - "state dir + container writable rootfs (shared nix store excluded); sampled every few minutes"; + "state/ + work/ + container writable rootfs (shared nix store excluded); sampled every few minutes"; disk.textContent = r.disk_bytes != null ? cloadFmtBytes(Number(r.disk_bytes)) : "—"; tr.append(disk); diff --git a/hive-c0re/src/lifecycle/host_config.rs b/hive-c0re/src/lifecycle/host_config.rs index a65df250..b8c2364c 100644 --- a/hive-c0re/src/lifecycle/host_config.rs +++ b/hive-c0re/src/lifecycle/host_config.rs @@ -148,7 +148,7 @@ fn config_bind_source(name: &str) -> PathBuf { /// The only caller left is the `ManageRootAgent` grant; the parent/child /// tree that handed every agent its own children here is gone. /// -/// **Three dirs, three different answers** — the uniformity of the +/// **Four dirs, three different answers** — the uniformity of the /// original loop is what hid that: /// /// - `state` is **read-write**: the holder reads and writes another @@ -164,6 +164,9 @@ fn config_bind_source(name: &str) -> PathBuf { /// which the holder has a stated reason to read, let alone write. /// hive-c0re still reads it directly on the host (`stats::hive_stats`), /// which needs no bind mount into anyone else's container. +/// - `work` is **absent entirely**, like `harness`. It is the agent's own +/// worktrees and build trees, often very large, and nothing gives the +/// holder a reason to see it. /// /// ⚠️ The config-repo seeding hive-c0re does at spawn is **not** affected by /// the `config` flag and must not be read as a reason to widen it: that runs @@ -567,6 +570,24 @@ mod tests { assert!(work_dir.is_dir()); } + /// `work` holds the agent's worktrees and build trees; like `harness` it + /// is never bound into another agent's container. + #[test] + fn a_holder_never_sees_another_agents_work_dir() { + for bind in child_binds() { + assert!( + !bind.container_path.contains("work"), + "another agent's work dir must not be bound in: {}", + bind.container_path + ); + assert!( + !bind.host_path.ends_with("/work"), + "another agent's work dir must not be bound in: {}", + bind.host_path + ); + } + } + /// An unparseable name yields no mounts at all — the guard must fail /// closed, since the alternative is a path built from unvalidated /// input. diff --git a/hive-c0re/src/main.rs b/hive-c0re/src/main.rs index 34eddc09..d8c4ef0f 100644 --- a/hive-c0re/src/main.rs +++ b/hive-c0re/src/main.rs @@ -399,7 +399,7 @@ async fn cmd_serve( // (turn-stats.sqlite has no vacuum — it's one tiny row per turn, // ~hundreds of KB, and the /stats + hive-stats views read it // directly; pruning it would just lose trend history for no gain.) - // Slow per-container disk sampler: a `du` of each agent's state dir + // Slow per-container disk sampler: a `du` of each agent's state/ + work/ // + writable rootfs every ~5 min, cached so the 5s container-load // poll stays cheap cgroup-only reads. Feeds `disk_bytes` on the LOAD // tab. See container_stats::disk_sampler_loop. diff --git a/hive-c0re/src/stats/container_stats.rs b/hive-c0re/src/stats/container_stats.rs index 1ba5c0e9..37329e4b 100644 --- a/hive-c0re/src/stats/container_stats.rs +++ b/hive-c0re/src/stats/container_stats.rs @@ -57,7 +57,7 @@ pub struct ContainerResource { /// Memory ceiling (`memory.max`), bytes. `None` when unlimited /// (the file reads `max`). pub mem_max_bytes: Option, - /// Last-sampled total disk usage — the agent's state dir plus the + /// Last-sampled total disk usage — the agent's `state/` and `work/` dirs plus the /// container's writable rootfs, excluding the shared read-only nix /// store — in bytes. `None` until the slow background sampler /// ([`disk_sampler_loop`]) has populated the cache at least once. diff --git a/nix/agent-modules/user.nix b/nix/agent-modules/user.nix index a044ddd6..ee083159 100644 --- a/nix/agent-modules/user.nix +++ b/nix/agent-modules/user.nix @@ -210,7 +210,9 @@ in chown -hR "$userName:$userName" "/agents/$userName/harness" 2>/dev/null || true fi # work/ holds large build trees and the agent creates everything in - # it, so only an ownership mismatch on the top dir needs a full walk. + # it, so a full walk is only needed when the top dir is not already + # the agent's: that covers a dir hive-c0re just created as hive-core + # and an agent uid that changed. Otherwise activation does one stat. if [ -d "/agents/$userName/work" ]; then if [ "$(stat -c %U "/agents/$userName/work")" != "$userName" ]; then chown -hR "$userName:$userName" "/agents/$userName/work" 2>/dev/null || true