diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 51993a92..153a6d85 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -140,13 +140,29 @@ let # those subdirectories, so one reader covers the host and every container. hostJournalDir = "/var/log/journal"; + # The operator-configured upstream, named once: the same exporter carries + # every signal, so metrics and logs both reach it without a second + # definition. + upstreamExporterName = if otelCfg.protocol == "grpc" then "otlp" else "otlphttp"; + upstreamExporters = lib.optional (otelCfg.endpoint != "") upstreamExporterName; + # One list, read by every pipeline: the per-hive pipelines fan out to # exactly the same destinations as the single pipeline they replace. # Written once because "which exporters" is a property of this tier, not # of which hive a sample came from. - exporterNames = - lib.optional (otelCfg.endpoint != "") (if otelCfg.protocol == "grpc" then "otlp" else "otlphttp") - ++ lib.optional vmCfg.enable "otlphttp/victoriametrics"; + exporterNames = upstreamExporters ++ lib.optional vmCfg.enable "otlphttp/victoriametrics"; + + # The same fan-out for logs, and the local store is only ONE of its + # destinations. A deployment that turns the swarm's log store off and keeps + # an upstream endpoint still collects — the store is where logs may be kept, + # not the reason to read the journal at all. + logExporterNames = upstreamExporters ++ lib.optional vlCfg.enable "otlphttp/victorialogs"; + + # Collect when there is anywhere to send it, and only then. A pipeline with + # an empty exporter list is not a quiet no-op — the collector rejects it — + # and reading the journal to drop it on the floor would be worse than not + # reading it. + collectLogs = logExporterNames != [ ]; in { options.services.hyperhive.swarm.otel = { @@ -672,7 +688,7 @@ in # write to. Read-only is the whole security posture of this mount: the # collector has no business writing to a journal, and a collector that # cannot write cannot corrupt the record it is reporting on. - // lib.optionalAttrs vlCfg.enable { + // lib.optionalAttrs collectLogs { ${hostJournalDir} = { hostPath = hostJournalDir; isReadOnly = true; @@ -811,7 +827,7 @@ in # are written by journald rather than by the logging process, so # a reader can tell the origins apart without this collector # deciding for them. - // lib.optionalAttrs vlCfg.enable { + // lib.optionalAttrs collectLogs { journald.directory = hostJournalDir; }; @@ -926,19 +942,20 @@ in exporters = exporterNames; }; } - # Logs go to the swarm's store and NOT to `exporterNames`, so - # they do not follow the metrics upstream. That asymmetry is - # deliberate: an operator who configured an upstream endpoint - # agreed to ship metrics there, and quietly adding their logs to - # the same hop is a disclosure decision, not a symmetry fix. + # Logs fan out exactly as metrics do: the swarm's store when it + # runs, the operator's upstream when one is configured, both + # when both. The local store is a destination rather than the + # reason to collect, so turning it off leaves a hive that still + # ships its journal to whoever aggregates across swarms. + # # Same stamp as the scraped swarm services, for the same reason: # a host's logs belong to the swarm, and there is no honest # `hive` value to put on them. - // lib.optionalAttrs vlCfg.enable { + // lib.optionalAttrs collectLogs { "logs/${swarmTierName}" = { receivers = [ "journald" ]; processors = [ "resource/${swarmTierName}" ]; - exporters = [ "otlphttp/victorialogs" ]; + exporters = logExporterNames; }; }; } @@ -1030,7 +1047,7 @@ in # a collector that refuses to start, and enabling the log store # without declaring a scrape target is a perfectly ordinary # config. - // lib.optionalAttrs (cfg.scrapeTargets != { } || vlCfg.enable) { + // lib.optionalAttrs (cfg.scrapeTargets != { } || collectLogs) { "resource/${swarmTierName}".attributes = [ { # The metric LABEL, a different namespace from the @@ -1072,7 +1089,7 @@ in # host. A dynamically allocated gid would mean the host's ownership # named a different group inside the container, and the failure # would be a receiver that starts cleanly and reads nothing. - // lib.optionalAttrs vlCfg.enable { + // lib.optionalAttrs collectLogs { SupplementaryGroups = [ "systemd-journal" ]; }; };