diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 32ba6eba..b79258aa 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -1106,10 +1106,11 @@ in # switch, and its `matches` field is an allowlist too, so # "everything except user sessions" is not expressible. # - # Attribution is journald's either way — `_HOSTNAME`, - # `_SYSTEMD_UNIT` and `_MACHINE_ID` are written by journald - # rather than by the logging process, so a reader can still - # tell the origins apart. + # Attribution is journald's either way — these fields are + # written by it rather than by the logging process. ⚠️ But + # `_MACHINE_ID` is the only one that separates containers: + # none sets `networking.hostName`, so `_HOSTNAME` is the + # NixOS default `nixos` everywhere but the host itself. // lib.optionalAttrs collectLogs { journald = { directory = hostJournalDir; @@ -1156,10 +1157,15 @@ in # fields, so without `_msg_field` every record stores the # literal `missing _msg field` as its message and text search # finds nothing, while ingest keeps answering 200. - # `_stream_fields` is one stream per unit per machine instead - # of one for the whole host. + # `_stream_fields` is one stream per unit per machine. + # `_MACHINE_ID` is what makes that true: `_HOSTNAME` reads + # `nixos` in every container, so naming it alone merges every + # container's stream for a unit name into one — which is how + # two collectors sharing `opentelemetry-collector.service` + # became one series. "otlphttp/victorialogs" = { - logs_endpoint = logsPushUrl + "?_msg_field=MESSAGE&_stream_fields=_HOSTNAME,_SYSTEMD_UNIT"; + logs_endpoint = + logsPushUrl + "?_msg_field=MESSAGE&_stream_fields=_HOSTNAME,_MACHINE_ID,_SYSTEMD_UNIT"; } // lib.optionalAttrs haveCollectorSecret { auth.authenticator = pushAuthenticator "victorialogs"; diff --git a/nix/module-eval.nix b/nix/module-eval.nix index aab24f7e..7111151b 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -1437,6 +1437,18 @@ let && lib.hasInfix "metrics.t.local" m && lib.hasInfix "logs.t.local" l; } + { + # `_HOSTNAME` cannot separate containers: none sets + # `networking.hostName`, so it is `nixos` for all of them and every + # container's stream for a unit name merges into one. + name = "the log stream is keyed by machine, not only by a hostname every container shares"; + ok = + let + l = (otelSettings otelNoStores).exporters."otlphttp/victorialogs".logs_endpoint; + field = f: lib.hasInfix ("_stream_fields=" + f) l || lib.hasInfix ("," + f) l; + in + field "_MACHINE_ID" && field "_SYSTEMD_UNIT" && !(field "_NOSUCHFIELD"); + } { # The collector reaches these routes through the gateway now, so each # store needs an ingest location of its own. Without one the write rides