From d9f09f4a52f95c445fb4efe9ff6cc1b7e589ed11 Mon Sep 17 00:00:00 2001
From: atlas <atlas@hyperhive.local>
Date: Wed, 26 Aug 2026 17:57:26 +0200
Subject: [PATCH] otel: collect host metrics on the hive-tier collector

Nothing measured the host itself. hive-c0re emits per-container metrics
and each service exports its own, so the store could answer 'is this
container using more memory than last week' and could not answer 'is the
machine under them out of memory' -- the question every one of those
readings is implicitly relative to.

Hive tier rather than swarm: a host is owned by the hive running on it,
and the swarm tier runs one collector for the whole swarm, so it would
have to reach other machines to see theirs.

Every scraper the pinned receiver offers except process, which adds a
series set per running process -- unbounded on a hive host, where the
nine shipped are a fixed handful. Coverage and cardinality are different
axes and only the second can hurt the store.

Refs #3649
---
 nix/host-modules/otel.nix | 55 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 54 insertions(+), 1 deletion(-)

diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix
index a57af75e..f41216e4 100644
--- a/nix/host-modules/otel.nix
+++ b/nix/host-modules/otel.nix
@@ -349,6 +349,51 @@ in
         settings = {
           receivers = {
             otlp.protocols.http.endpoint = listen;
+
+            # The host itself, which nothing else measures. hive-c0re emits
+            # per-container metrics and each service exports its own, so the
+            # store could answer "is this container using more memory than
+            # last week" and could not answer "is the machine under them out
+            # of memory" — the question every one of those readings is
+            # implicitly relative to.
+            #
+            # Here rather than in ./swarm-otel.nix because a host is owned by
+            # the hive running on it: the swarm tier runs one collector for
+            # the whole swarm and would have to reach *other* machines to see
+            # theirs. Consequence worth knowing before reading a dashboard:
+            # these samples carry `hive` and no `agent`, so "which host" is
+            # answered by "which hive" — true only while a hive is one host.
+            #
+            # Every scraper the pinned receiver offers except `process`, and
+            # the exception is not squeamishness about volume: `process` adds
+            # a series set PER RUNNING PROCESS, which on a hive host means
+            # every agent's claude and every nix build. That is unbounded and
+            # churning, where the nine below are a fixed handful per host
+            # forever. Coverage and cardinality are different axes and only
+            # the second one can hurt the store.
+            #
+            # Names verified against THIS collector version rather than
+            # upstream's docs, with a deliberately bogus scraper rejected in
+            # the same run so that "accepted" is distinguishable from "the
+            # validator ignores unknown keys". `validateConfigFile` above
+            # keeps that honest at build time.
+            hostmetrics = {
+              # Stated rather than inherited: the upstream default is a
+              # value that can change under a version bump, and the sample
+              # interval is the one knob here with a storage bill attached.
+              collection_interval = "60s";
+              scrapers = {
+                cpu = { };
+                memory = { };
+                filesystem = { };
+                load = { };
+                disk = { };
+                network = { };
+                paging = { };
+                processes = { };
+                system = { };
+              };
+            };
           }
           # Only when a service has actually declared a target. An enabled
           # `prometheus` receiver with an empty `scrape_configs` renders,
@@ -407,7 +452,15 @@ in
             # `optional` matters too: a pipeline naming a receiver that no
             # longer exists is a startup failure, which `validateConfigFile`
             # below turns into a build failure rather than a broken deploy.
-            receivers = [ "otlp" ] ++ lib.optional (otel.scrapeTargets != { }) "prometheus";
+            # `hostmetrics` is unconditional, unlike `prometheus` above: it
+            # has no per-hive configuration to be empty of, so there is no
+            # inert shape to guard against — the receiver either collects the
+            # host's metrics or the host has none collected at all.
+            receivers = [
+              "otlp"
+              "hostmetrics"
+            ]
+            ++ lib.optional (otel.scrapeTargets != { }) "prometheus";
             exporters = [ swarmName ];
           };
         }