diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index b79258aa..f3294def 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -1107,10 +1107,12 @@ in # "everything except user sessions" is not expressible. # # Attribution is journald's either way — these fields are - # written by it rather than by the logging process. ⚠️ But - # `_MACHINE_ID` is the only one that separates containers: - # none sets `networking.hostName`, so `_HOSTNAME` is the - # NixOS default `nixos` everywhere but the host itself. + # written by it rather than by the logging process. ⚠️ Use + # `_MACHINE_ID` to tell machines apart, not `_HOSTNAME`: an + # agent container reports the NixOS default `nixos`. Whether + # every container does, and why the `networking.hostName` + # nixos-containers sets does not reach them, is unresolved — + # so rely on the field that is per-machine by construction. // lib.optionalAttrs collectLogs { journald = { directory = hostJournalDir; @@ -1158,11 +1160,13 @@ in # literal `missing _msg field` as its message and text search # finds nothing, while ingest keeps answering 200. # `_stream_fields` is one stream per unit per machine. - # `_MACHINE_ID` is what makes that true: `_HOSTNAME` reads - # `nixos` in every container, so naming it alone merges every - # container's stream for a unit name into one — which is how - # two collectors sharing `opentelemetry-collector.service` - # became one series. + # `_MACHINE_ID` is what makes that true — it is per-machine by + # construction, where `_HOSTNAME` is only as distinct as the + # hostnames happen to be, and an agent container's is the + # NixOS default `nixos`. Keying on the hostname alone merges + # the streams of any two machines sharing one, which is what + # the host and containerised `opentelemetry-collector.service` + # would do. "otlphttp/victorialogs" = { logs_endpoint = logsPushUrl + "?_msg_field=MESSAGE&_stream_fields=_HOSTNAME,_MACHINE_ID,_SYSTEMD_UNIT";