From d9f09f4a52f95c445fb4efe9ff6cc1b7e589ed11 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 26 Aug 2026 17:57:26 +0200 Subject: [PATCH 1/2] otel: collect host metrics on the hive-tier collector Nothing measured the host itself. hive-c0re emits per-container metrics and each service exports its own, so the store could answer 'is this container using more memory than last week' and could not answer 'is the machine under them out of memory' -- the question every one of those readings is implicitly relative to. Hive tier rather than swarm: a host is owned by the hive running on it, and the swarm tier runs one collector for the whole swarm, so it would have to reach other machines to see theirs. Every scraper the pinned receiver offers except process, which adds a series set per running process -- unbounded on a hive host, where the nine shipped are a fixed handful. Coverage and cardinality are different axes and only the second can hurt the store. Refs #3649 --- nix/host-modules/otel.nix | 55 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index a57af75e..f41216e4 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -349,6 +349,51 @@ in settings = { receivers = { otlp.protocols.http.endpoint = listen; + + # The host itself, which nothing else measures. hive-c0re emits + # per-container metrics and each service exports its own, so the + # store could answer "is this container using more memory than + # last week" and could not answer "is the machine under them out + # of memory" — the question every one of those readings is + # implicitly relative to. + # + # Here rather than in ./swarm-otel.nix because a host is owned by + # the hive running on it: the swarm tier runs one collector for + # the whole swarm and would have to reach *other* machines to see + # theirs. Consequence worth knowing before reading a dashboard: + # these samples carry `hive` and no `agent`, so "which host" is + # answered by "which hive" — true only while a hive is one host. + # + # Every scraper the pinned receiver offers except `process`, and + # the exception is not squeamishness about volume: `process` adds + # a series set PER RUNNING PROCESS, which on a hive host means + # every agent's claude and every nix build. That is unbounded and + # churning, where the nine below are a fixed handful per host + # forever. Coverage and cardinality are different axes and only + # the second one can hurt the store. + # + # Names verified against THIS collector version rather than + # upstream's docs, with a deliberately bogus scraper rejected in + # the same run so that "accepted" is distinguishable from "the + # validator ignores unknown keys". `validateConfigFile` above + # keeps that honest at build time. + hostmetrics = { + # Stated rather than inherited: the upstream default is a + # value that can change under a version bump, and the sample + # interval is the one knob here with a storage bill attached. + collection_interval = "60s"; + scrapers = { + cpu = { }; + memory = { }; + filesystem = { }; + load = { }; + disk = { }; + network = { }; + paging = { }; + processes = { }; + system = { }; + }; + }; } # Only when a service has actually declared a target. An enabled # `prometheus` receiver with an empty `scrape_configs` renders, @@ -407,7 +452,15 @@ in # `optional` matters too: a pipeline naming a receiver that no # longer exists is a startup failure, which `validateConfigFile` # below turns into a build failure rather than a broken deploy. - receivers = [ "otlp" ] ++ lib.optional (otel.scrapeTargets != { }) "prometheus"; + # `hostmetrics` is unconditional, unlike `prometheus` above: it + # has no per-hive configuration to be empty of, so there is no + # inert shape to guard against — the receiver either collects the + # host's metrics or the host has none collected at all. + receivers = [ + "otlp" + "hostmetrics" + ] + ++ lib.optional (otel.scrapeTargets != { }) "prometheus"; exporters = [ swarmName ]; }; } From d702eeffe16c5a93cd835bce5cec579211a4f09a Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 26 Aug 2026 18:09:24 +0200 Subject: [PATCH 2/2] otel: say which of processes/process is configured They differ by one letter and a grep for the shorter name matches both. The summary is what ships; the per-process scraper is deliberately absent, and a reader (human or grep) cannot tell from the diff alone. --- nix/host-modules/otel.nix | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index f41216e4..170ca5ea 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -390,6 +390,12 @@ in disk = { }; network = { }; paging = { }; + # ⚠️ `processes` is the SUMMARY -- how many are running, + # blocked, sleeping. A handful of series per host. Its + # singular sibling `process` is the per-process one and is + # deliberately absent above; the two differ by one letter and + # a grep for the shorter name matches both, so read the + # trailing `s` before concluding which is configured here. processes = { }; system = { }; };