From bb38e1967ce4b497936ab12566b8b8dfd20e00c5 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 12 Sep 2026 14:14:11 +0200 Subject: [PATCH] swarm-otel: key the log stream by machine, not by a hostname every container shares MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_stream_fields=_HOSTNAME,_SYSTEMD_UNIT` was chosen to give "one stream per unit per machine", which is what the comment above it claims. It does not: no container sets `networking.hostName`, so `_HOSTNAME` is the NixOS default `nixos` in every one of them, and every container's stream for a given unit name merges into a single series. Measured from inside an agent container: journalctl -n1 -o json -> "_MACHINE_ID":"5d1427ea…", "_HOSTNAME":"nixos" cat /etc/machine-id -> 5d1427ea… git grep -i hostname -- nix/ -> 20 hits, all public vhost names, zero `networking.hostName =` `_MACHINE_ID` is written by journald per machine and is already on every entry, so adding it to the stream key makes the partition what its own comment says it is. That merge is also why the host collector and the containerised one were one series: both log to `opentelemetry-collector.service`, and the field meant to separate them was a constant. The second comment touched here asserted a reader "can still tell the origins apart" from `_HOSTNAME`/`_SYSTEMD_UNIT`/`_MACHINE_ID`. True only of the last one; it now says so. Not a full fix for attribution: `_MACHINE_ID` is opaque hex and nothing maps it to a container name. Naming is the other half and is a separate change. Refs #4304 --- nix/host-modules/swarm-otel.nix | 20 +++++++++++++------- nix/module-eval.nix | 12 ++++++++++++ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 32ba6eba..b79258aa 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -1106,10 +1106,11 @@ in # switch, and its `matches` field is an allowlist too, so # "everything except user sessions" is not expressible. # - # Attribution is journald's either way — `_HOSTNAME`, - # `_SYSTEMD_UNIT` and `_MACHINE_ID` are written by journald - # rather than by the logging process, so a reader can still - # tell the origins apart. + # Attribution is journald's either way — these fields are + # written by it rather than by the logging process. ⚠️ But + # `_MACHINE_ID` is the only one that separates containers: + # none sets `networking.hostName`, so `_HOSTNAME` is the + # NixOS default `nixos` everywhere but the host itself. // lib.optionalAttrs collectLogs { journald = { directory = hostJournalDir; @@ -1156,10 +1157,15 @@ in # fields, so without `_msg_field` every record stores the # literal `missing _msg field` as its message and text search # finds nothing, while ingest keeps answering 200. - # `_stream_fields` is one stream per unit per machine instead - # of one for the whole host. + # `_stream_fields` is one stream per unit per machine. + # `_MACHINE_ID` is what makes that true: `_HOSTNAME` reads + # `nixos` in every container, so naming it alone merges every + # container's stream for a unit name into one — which is how + # two collectors sharing `opentelemetry-collector.service` + # became one series. "otlphttp/victorialogs" = { - logs_endpoint = logsPushUrl + "?_msg_field=MESSAGE&_stream_fields=_HOSTNAME,_SYSTEMD_UNIT"; + logs_endpoint = + logsPushUrl + "?_msg_field=MESSAGE&_stream_fields=_HOSTNAME,_MACHINE_ID,_SYSTEMD_UNIT"; } // lib.optionalAttrs haveCollectorSecret { auth.authenticator = pushAuthenticator "victorialogs"; diff --git a/nix/module-eval.nix b/nix/module-eval.nix index aab24f7e..7111151b 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -1437,6 +1437,18 @@ let && lib.hasInfix "metrics.t.local" m && lib.hasInfix "logs.t.local" l; } + { + # `_HOSTNAME` cannot separate containers: none sets + # `networking.hostName`, so it is `nixos` for all of them and every + # container's stream for a unit name merges into one. + name = "the log stream is keyed by machine, not only by a hostname every container shares"; + ok = + let + l = (otelSettings otelNoStores).exporters."otlphttp/victorialogs".logs_endpoint; + field = f: lib.hasInfix ("_stream_fields=" + f) l || lib.hasInfix ("," + f) l; + in + field "_MACHINE_ID" && field "_SYSTEMD_UNIT" && !(field "_NOSUCHFIELD"); + } { # The collector reaches these routes through the gateway now, so each # store needs an ingest location of its own. Without one the write rides