diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 4c8cc955..fddaf4c7 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -32,6 +32,7 @@ let swarmCfg = config.services.hyperhive.swarm; otelCfg = config.services.hyperhive.otel; vmCfg = config.services.hyperhive.swarm.victoriametrics; + vlCfg = config.services.hyperhive.swarm.victorialogs; hyperhiveCfg = config.services.hyperhive; gatewayCfg = hyperhiveCfg.gateway; swarmDomain = hyperhiveCfg.swarm.domain; @@ -127,6 +128,18 @@ let # both "the label is absent" and "the label says unknown". swarmDisplayName = if hyperhiveCfg.swarm.name == null then "unknown" else hyperhiveCfg.swarm.name; + # Where the journal lives on BOTH sides of the bind mount below — one string + # because the receiver reads the path it is mounted at, and two spellings of + # one path is a mount that succeeds and a receiver that finds nothing. + # + # 🔑 Why the host's directory is enough to see every container: nspawn is + # invoked with `--link-journal=try-guest` for every non-ephemeral container, + # so a container's journal FILES live here, under its own machine-id + # subdirectory, and are bind-mounted into the guest rather than the other way + # round. Measured: `journalctl -D` on this parent directory descends into + # those subdirectories, so one reader covers the host and every container. + hostJournalDir = "/var/log/journal"; + # One list, read by every pipeline: the per-hive pipelines fan out to # exactly the same destinations as the single pipeline they replace. # Written once because "which exporters" is a property of this tier, not @@ -654,7 +667,17 @@ in } # The public hive CA, read-only — only when something in here # actually verifies a swarm-service name. - // caTrust.bindMount; + // caTrust.bindMount + # The host's journal, read-only, and only when there is a log store to + # write to. Read-only is the whole security posture of this mount: the + # collector has no business writing to a journal, and a collector that + # cannot write cannot corrupt the record it is reporting on. + // lib.optionalAttrs vlCfg.enable { + ${hostJournalDir} = { + hostPath = hostJournalDir; + isReadOnly = true; + }; + }; config = { ... }: @@ -772,6 +795,24 @@ in }; } ) cfg.publishedScrapeTargets; + } + # The whole journal directory, deliberately unfiltered. + # + # A filter restricted to the swarm's own containers was the + # obvious shape and is the wrong one: the host units are where + # the incidents live — the gateway's nginx, the core daemon, + # dnsmasq — and none of them is a swarm container. Filtering to + # `swarm-*` would exclude the single most-needed source. + # + # Nor would a list of container names stay true: it is a second + # place that has to know which services exist, and it goes stale + # silently the next time one is added. Attribution does not need + # it — journald's `_HOSTNAME` / `_SYSTEMD_UNIT` / `_MACHINE_ID` + # are written by journald rather than by the logging process, so + # a reader can tell the origins apart without this collector + # deciding for them. + // lib.optionalAttrs vlCfg.enable { + journald.directory = hostJournalDir; }; exporters = @@ -797,6 +838,22 @@ in headers.${otelCfg.collector.upstreamHeaderName} = "\${env:${otelCfg.collector.upstreamHeaderName}}"; } // lib.optionalAttrs (otelCfg.protocol == "http/json") { encoding = "json"; }; + } + // lib.optionalAttrs vlCfg.enable { + # `logs_endpoint`, NOT `endpoint`, for exactly the reason the + # metrics exporter above spells out — and the trap is worse + # here, because the two stores' OTLP routes differ. `endpoint` + # is a base the exporter appends `/v1/logs` to; VictoriaLogs + # serves `/insert/opentelemetry/v1/logs`, which is also not + # the metrics store's `/opentelemetry/api/v1/push`. Both + # spellings pass `otelcol validate`. + # + # ⚠️ And the store cannot tell you either: a right path, a + # wrong path and a nonsense path all answer 400. Only its log + # distinguishes them — the real route complains about the + # encoding, everything else says "unsupported path requested". + "otlphttp/victorialogs".logs_endpoint = + "http://127.0.0.1:${toString vlCfg.port}/insert/opentelemetry/v1/logs"; }; # Moves this collector's self-metrics off the built-in @@ -850,6 +907,21 @@ in processors = [ "resource/${swarmTierName}" ]; exporters = exporterNames; }; + } + # Logs go to the swarm's store and NOT to `exporterNames`, so + # they do not follow the metrics upstream. That asymmetry is + # deliberate: an operator who configured an upstream endpoint + # agreed to ship metrics there, and quietly adding their logs to + # the same hop is a disclosure decision, not a symmetry fix. + # Same stamp as the scraped swarm services, for the same reason: + # a host's logs belong to the swarm, and there is no honest + # `hive` value to put on them. + // lib.optionalAttrs vlCfg.enable { + "logs/${swarmTierName}" = { + receivers = [ "journald" ]; + processors = [ "resource/${swarmTierName}" ]; + exporters = [ "otlphttp/victorialogs" ]; + }; }; } // { @@ -934,7 +1006,13 @@ in # any one hive, so there is no honest value to put there — and # an invented one (a sentinel, the local hive's name) would be # queried as though it meant something. - // lib.optionalAttrs (cfg.scrapeTargets != { }) { + # + # ⚠️ Emitted for the LOG pipeline too, not just the scraped one: + # a processor a pipeline names but the config does not define is + # a collector that refuses to start, and enabling the log store + # without declaring a scrape target is a perfectly ordinary + # config. + // lib.optionalAttrs (cfg.scrapeTargets != { } || vlCfg.enable) { "resource/${swarmTierName}".attributes = [ { # The metric LABEL, a different namespace from the @@ -966,6 +1044,18 @@ in # scrapes nothing published — the empty case is the shipped one. // lib.optionalAttrs (cfg.publishedScrapeTargets != { }) { LoadCredential = [ "${collectorCredentialId}:${collectorSecretInContainer}" ]; + } + # Journal files are `0640 root:systemd-journal`, and this unit runs + # with upstream's `DynamicUser = true` — so there is no stable uid to + # grant, and the group is the only way in. + # + # 🔑 That works across the bind mount because `systemd-journal` is + # gid **62**, FIXED in nixpkgs' `ids.nix` rather than allocated per + # host. A dynamically allocated gid would mean the host's ownership + # named a different group inside the container, and the failure + # would be a receiver that starts cleanly and reads nothing. + // lib.optionalAttrs vlCfg.enable { + SupplementaryGroups = [ "systemd-journal" ]; }; }; };