diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index a57af75e..f41216e4 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -349,6 +349,51 @@ in settings = { receivers = { otlp.protocols.http.endpoint = listen; + + # The host itself, which nothing else measures. hive-c0re emits + # per-container metrics and each service exports its own, so the + # store could answer "is this container using more memory than + # last week" and could not answer "is the machine under them out + # of memory" — the question every one of those readings is + # implicitly relative to. + # + # Here rather than in ./swarm-otel.nix because a host is owned by + # the hive running on it: the swarm tier runs one collector for + # the whole swarm and would have to reach *other* machines to see + # theirs. Consequence worth knowing before reading a dashboard: + # these samples carry `hive` and no `agent`, so "which host" is + # answered by "which hive" — true only while a hive is one host. + # + # Every scraper the pinned receiver offers except `process`, and + # the exception is not squeamishness about volume: `process` adds + # a series set PER RUNNING PROCESS, which on a hive host means + # every agent's claude and every nix build. That is unbounded and + # churning, where the nine below are a fixed handful per host + # forever. Coverage and cardinality are different axes and only + # the second one can hurt the store. + # + # Names verified against THIS collector version rather than + # upstream's docs, with a deliberately bogus scraper rejected in + # the same run so that "accepted" is distinguishable from "the + # validator ignores unknown keys". `validateConfigFile` above + # keeps that honest at build time. + hostmetrics = { + # Stated rather than inherited: the upstream default is a + # value that can change under a version bump, and the sample + # interval is the one knob here with a storage bill attached. + collection_interval = "60s"; + scrapers = { + cpu = { }; + memory = { }; + filesystem = { }; + load = { }; + disk = { }; + network = { }; + paging = { }; + processes = { }; + system = { }; + }; + }; } # Only when a service has actually declared a target. An enabled # `prometheus` receiver with an empty `scrape_configs` renders, @@ -407,7 +452,15 @@ in # `optional` matters too: a pipeline naming a receiver that no # longer exists is a startup failure, which `validateConfigFile` # below turns into a build failure rather than a broken deploy. - receivers = [ "otlp" ] ++ lib.optional (otel.scrapeTargets != { }) "prometheus"; + # `hostmetrics` is unconditional, unlike `prometheus` above: it + # has no per-hive configuration to be empty of, so there is no + # inert shape to guard against — the receiver either collects the + # host's metrics or the host has none collected at all. + receivers = [ + "otlp" + "hostmetrics" + ] + ++ lib.optional (otel.scrapeTargets != { }) "prometheus"; exporters = [ swarmName ]; }; }