From daea908d698f41656bd700e7a69d1b3aff07a621 Mon Sep 17 00:00:00 2001
From: atlas <atlas@hyperhive.local>
Date: Thu, 27 Aug 2026 10:48:46 +0200
Subject: [PATCH] otel: give host metrics a host identity via resourcedetection

Host metrics arrived carrying `hive` and no host attribute, so "which
host is out of memory" was answerable only as "which hive" -- true only
while a hive is one machine, which is the framing the swarm work exists
to end.

Adds the collector's own `resourcedetection` processor to the metrics
pipeline, emitting both `host.name` and `host.id`.

`resourcedetection` rather than a value picked in nix: this is the
canonical OTel mechanism for host identity, and choosing our own would
answer a question the tool already answers, differently from every other
deployment. A resource attribute is captured into the stored series, so
a private naming scheme is expensive in the way that lasts.

Both attributes, deliberately: `host.name` is readable and what a panel
groups by, but splits the series silently on a rename with nothing
linking old to new; `host.id` survives that and is unreadable alone.
Hosts get renamed and repurposed exactly when services move off one box.

The processor sits in the pipeline, so it applies to every receiver --
agent-pushed OTLP and scraped targets included, not just `hostmetrics`.

Keys verified against otelcol-contrib 0.151.0 with two deliberate
mutations rejected in the same run (a bogus resource attribute and a
bogus detector key), so "accepted" is distinguishable from a validator
that ignores what it does not recognise.
---
 nix/host-modules/otel.nix | 46 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix
index 170ca5ea..40b9e1b8 100644
--- a/nix/host-modules/otel.nix
+++ b/nix/host-modules/otel.nix
@@ -450,6 +450,44 @@ in
           # clean and the exporter naming it sends nothing authenticated.
           service.extensions = lib.optional senderAuth authName;
 
+          # Host identity, so a sample can say WHICH machine it came from.
+          # Without this the host's metrics carry `hive` and no host
+          # attribute, so "which host" is answered by "which hive" — true
+          # only while a hive is one machine, which is exactly the framing
+          # the swarm work exists to end.
+          #
+          # `resourcedetection` rather than a value chosen in nix: this is
+          # the collector's own canonical mechanism for host identity, and
+          # picking our own would answer a question the tool already
+          # answers, differently from every other OTel deployment. A
+          # resource attribute is CAPTURED into the stored series, so a
+          # private naming scheme is expensive in the one way that lasts —
+          # every historical comparison keys on it.
+          #
+          # Keys verified against the collector binary rather than
+          # upstream's docs, with two deliberate mutations rejected in the
+          # same run — a bogus resource attribute and a bogus detector key
+          # — so "accepted" is distinguishable from a validator that
+          # ignores what it does not recognise. `validateConfigFile` above
+          # keeps that true at build time against the pinned version.
+          processors.resourcedetection = {
+            detectors = [ "system" ];
+
+            # BOTH attributes, deliberately:
+            #   host.name — readable, and what a panel groups by
+            #   host.id   — /etc/machine-id, survives a rename
+            # A name-keyed history splits silently the moment a machine is
+            # renamed or repurposed, with nothing linking old to new — and
+            # hosts get renamed and repurposed precisely when services move
+            # off one box. Neither alone is sufficient: an id nobody can
+            # read is not a dashboard, and a name that moves is not
+            # history. Cost is one extra label per series.
+            system.resource_attributes = {
+              "host.name".enabled = true;
+              "host.id".enabled = true;
+            };
+          };
+
           service.pipelines.metrics = {
             # EXTENDED, not replaced. Assigning here instead of appending
             # would drop `otlp` — the hive would stop receiving from its
@@ -467,6 +505,14 @@ in
               "hostmetrics"
             ]
             ++ lib.optional (otel.scrapeTargets != { }) "prometheus";
+            # Applies to EVERY receiver above, not just `hostmetrics` — a
+            # processor sits in the pipeline, not on a receiver. That is
+            # what we want here: samples an agent pushed over `otlp` and
+            # samples scraped from a target both arrive at the store with
+            # the identity of the machine that collected them, and a
+            # reading is only comparable across hives once it says where
+            # it came from.
+            processors = [ "resourcedetection" ];
             exporters = [ swarmName ];
           };
         }