diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 32ba6eba..f3294def 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -1106,10 +1106,13 @@ in # switch, and its `matches` field is an allowlist too, so # "everything except user sessions" is not expressible. # - # Attribution is journald's either way — `_HOSTNAME`, - # `_SYSTEMD_UNIT` and `_MACHINE_ID` are written by journald - # rather than by the logging process, so a reader can still - # tell the origins apart. + # Attribution is journald's either way — these fields are + # written by it rather than by the logging process. ⚠️ Use + # `_MACHINE_ID` to tell machines apart, not `_HOSTNAME`: an + # agent container reports the NixOS default `nixos`. Whether + # every container does, and why the `networking.hostName` + # nixos-containers sets does not reach them, is unresolved — + # so rely on the field that is per-machine by construction. // lib.optionalAttrs collectLogs { journald = { directory = hostJournalDir; @@ -1156,10 +1159,17 @@ in # fields, so without `_msg_field` every record stores the # literal `missing _msg field` as its message and text search # finds nothing, while ingest keeps answering 200. - # `_stream_fields` is one stream per unit per machine instead - # of one for the whole host. + # `_stream_fields` is one stream per unit per machine. + # `_MACHINE_ID` is what makes that true — it is per-machine by + # construction, where `_HOSTNAME` is only as distinct as the + # hostnames happen to be, and an agent container's is the + # NixOS default `nixos`. Keying on the hostname alone merges + # the streams of any two machines sharing one, which is what + # the host and containerised `opentelemetry-collector.service` + # would do. "otlphttp/victorialogs" = { - logs_endpoint = logsPushUrl + "?_msg_field=MESSAGE&_stream_fields=_HOSTNAME,_SYSTEMD_UNIT"; + logs_endpoint = + logsPushUrl + "?_msg_field=MESSAGE&_stream_fields=_HOSTNAME,_MACHINE_ID,_SYSTEMD_UNIT"; } // lib.optionalAttrs haveCollectorSecret { auth.authenticator = pushAuthenticator "victorialogs"; diff --git a/nix/module-eval.nix b/nix/module-eval.nix index aab24f7e..7111151b 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -1437,6 +1437,18 @@ let && lib.hasInfix "metrics.t.local" m && lib.hasInfix "logs.t.local" l; } + { + # `_HOSTNAME` cannot separate containers: none sets + # `networking.hostName`, so it is `nixos` for all of them and every + # container's stream for a unit name merges into one. + name = "the log stream is keyed by machine, not only by a hostname every container shares"; + ok = + let + l = (otelSettings otelNoStores).exporters."otlphttp/victorialogs".logs_endpoint; + field = f: lib.hasInfix ("_stream_fields=" + f) l || lib.hasInfix ("," + f) l; + in + field "_MACHINE_ID" && field "_SYSTEMD_UNIT" && !(field "_NOSUCHFIELD"); + } { # The collector reaches these routes through the gateway now, so each # store needs an ingest location of its own. Without one the write rides