diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 170ca5ea..40b9e1b8 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -450,6 +450,44 @@ in # clean and the exporter naming it sends nothing authenticated. service.extensions = lib.optional senderAuth authName; + # Host identity, so a sample can say WHICH machine it came from. + # Without this the host's metrics carry `hive` and no host + # attribute, so "which host" is answered by "which hive" — true + # only while a hive is one machine, which is exactly the framing + # the swarm work exists to end. + # + # `resourcedetection` rather than a value chosen in nix: this is + # the collector's own canonical mechanism for host identity, and + # picking our own would answer a question the tool already + # answers, differently from every other OTel deployment. A + # resource attribute is CAPTURED into the stored series, so a + # private naming scheme is expensive in the one way that lasts — + # every historical comparison keys on it. + # + # Keys verified against the collector binary rather than + # upstream's docs, with two deliberate mutations rejected in the + # same run — a bogus resource attribute and a bogus detector key + # — so "accepted" is distinguishable from a validator that + # ignores what it does not recognise. `validateConfigFile` above + # keeps that true at build time against the pinned version. + processors.resourcedetection = { + detectors = [ "system" ]; + + # BOTH attributes, deliberately: + # host.name — readable, and what a panel groups by + # host.id — /etc/machine-id, survives a rename + # A name-keyed history splits silently the moment a machine is + # renamed or repurposed, with nothing linking old to new — and + # hosts get renamed and repurposed precisely when services move + # off one box. Neither alone is sufficient: an id nobody can + # read is not a dashboard, and a name that moves is not + # history. Cost is one extra label per series. + system.resource_attributes = { + "host.name".enabled = true; + "host.id".enabled = true; + }; + }; + service.pipelines.metrics = { # EXTENDED, not replaced. Assigning here instead of appending # would drop `otlp` — the hive would stop receiving from its @@ -467,6 +505,14 @@ in "hostmetrics" ] ++ lib.optional (otel.scrapeTargets != { }) "prometheus"; + # Applies to EVERY receiver above, not just `hostmetrics` — a + # processor sits in the pipeline, not on a receiver. That is + # what we want here: samples an agent pushed over `otlp` and + # samples scraped from a target both arrive at the store with + # the identity of the machine that collected them, and a + # reading is only comparable across hives once it says where + # it came from. + processors = [ "resourcedetection" ]; exporters = [ swarmName ]; }; }