diff --git a/docs/4-implementation/scheduler/observability.md b/docs/4-implementation/scheduler/observability.md index 9aa7ccae..0c4125d3 100644 --- a/docs/4-implementation/scheduler/observability.md +++ b/docs/4-implementation/scheduler/observability.md @@ -208,6 +208,12 @@ fixes the collector port every hive and its agents share, That's the reference; this page covers what a per-option listing can't: the two-tier architecture, the security model, and how the options interact. +The host collector's `hostmetrics` filesystem scraper reports only the mount +points in `services.hyperhive.otel.collector.filesystemMountPoints` (default +`/`, `/boot` and `/nix`), as `system.filesystem.*` with a `mountpoint` label. A +listed mount the host lacks reports nothing; `/nix` appears only where it's its +own mount. + ## Built-in resource labels The harness sets the OTLP variables (`OTEL_EXPORTER_OTLP_ENDPOINT`, `_PROTOCOL`, diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 8505eb9d..bfe7da4d 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -189,6 +189,29 @@ in ''; }; + collector.filesystemMountPoints = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ + "/" + "/boot" + "/nix" + ]; + example = [ + "/" + "/boot" + "/data" + ]; + description = '' + Mount points the host collector's `filesystem` scraper reports + (`system.filesystem.*`), matched exactly. The list is strict because + the scraper has to include virtual filesystems to see a btrfs + subvolume root, and that would otherwise report every tmpfs, proc + and cgroup mount plus each per-container btrfs subvolume, which + repeats the same device's usage. A mount point on the list that a + host does not have reports nothing. + ''; + }; + httpChecks = lib.mkOption { type = lib.types.listOf lib.types.str; default = [ ]; @@ -480,7 +503,19 @@ in scrapers = { cpu = { }; memory = { }; - filesystem = { }; + # `include_virtual_filesystems` turns gopsutil's `all` on. + # With it off, a mount whose mountinfo root is not `/` + # counts as a bind mount and is dropped, which includes a + # btrfs `subvol=@` root. The strict list then keeps the + # now-visible tmpfs, proc and per-container subvolume + # mounts out of the store. + filesystem = { + include_virtual_filesystems = true; + include_mount_points = { + match_type = "strict"; + mount_points = otel.collector.filesystemMountPoints; + }; + }; load = { }; disk = { }; network = { }; diff --git a/nix/module-eval/matrix-core.nix b/nix/module-eval/matrix-core.nix index 8bea9c56..6a392307 100644 --- a/nix/module-eval/matrix-core.nix +++ b/nix/module-eval/matrix-core.nix @@ -207,6 +207,31 @@ let && (settings matrixElsewhere).receivers ? hostmetrics && lib.elem "hostmetrics" (pipeline matrixElsewhere); } + { + # The root of a btrfs `subvol=@` host has a mountinfo root other than + # `/`, which the scraper drops as a bind mount unless virtual + # filesystems are included. The strict list is what keeps that from + # reporting every tmpfs and per-container subvolume. The lookup is + # read on a host that has the receiver (the pipeline arm above is its + # control). + name = "the host collector's filesystem scraper includes virtual filesystems and reports only the listed mount points"; + ok = + let + fs = + matrixElsewhere.services.opentelemetry-collector.settings.receivers.hostmetrics.scrapers.filesystem; + in + fs.include_virtual_filesystems == true + && fs.include_mount_points.match_type == "strict" + && + fs.include_mount_points.mount_points + == matrixElsewhere.services.hyperhive.otel.collector.filesystemMountPoints + && + fs.include_mount_points.mount_points == [ + "/" + "/boot" + "/nix" + ]; + } { # The size is read from the database directory, as the dynamic user the # homeserver runs as, and pushed to the forwarder's OTLP port. Without a