From 527e07c5e0e8ebe9419010292a53485e6c65b7aa Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 24 Aug 2026 16:36:18 +0200 Subject: [PATCH 1/6] feat(swarm-victorialogs): a log store for the swarm An agent can verify that a unit was launched and never that it is working: container journals are not reachable, so a diagnosis stops at the first broken component -- which is precisely the component whose own instrument is least likely to be legible. This is the store half of collecting logs centrally so the question becomes answerable. Mirrors swarm-victoriametrics deliberately: same container shape, same loopback pin, same self-scrape arrangement. Two differences, both intentional. No gateway vhost. VictoriaLogs' ingest and query endpoints carry no authentication of their own, exactly like the metrics store's -- and the metrics store IS published under a resolvable name, which is an open question rather than a settled design. Publishing this one the same way would repeat that before the first instance is decided. Retention defaults to 30d against the metrics store's 5y. Logs are orders of magnitude larger per unit of time and their value decays much faster: a log line answers what happened during an incident, a metric answers whether this is worse than last quarter. Not wired into enableRequiredServices yet -- that lands with the collector pipeline, so we do not start a store nothing writes to. --- nix/host-modules/default.nix | 1 + nix/host-modules/swarm-victorialogs.nix | 168 ++++++++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 nix/host-modules/swarm-victorialogs.nix diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index 666490b5..da0225c9 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -30,6 +30,7 @@ ./swarm-otel.nix ./swarm-snapshot-store.nix ./swarm-ui.nix + ./swarm-victorialogs.nix ./swarm-victoriametrics.nix ./swarm-wireguard.nix ./swarm.nix diff --git a/nix/host-modules/swarm-victorialogs.nix b/nix/host-modules/swarm-victorialogs.nix new file mode 100644 index 00000000..3e1e4781 --- /dev/null +++ b/nix/host-modules/swarm-victorialogs.nix @@ -0,0 +1,168 @@ +# The swarm's log store: one VictoriaLogs for the whole swarm, in a +# `swarm-victorialogs` nixos-container beside the metrics store it mirrors. +# +# Why a store at all rather than reading journals directly: an agent can +# verify that a unit was *launched* and never that it is *working*. The +# container journals are not reachable from an agent, so a diagnosis stops at +# the first component that is broken — which is precisely the component whose +# own instrument is least likely to be legible. Collecting logs centrally +# makes the question answerable without host access. +# +# It is the collector that feeds this, not the services directly: one ingest +# point per swarm, same shape as the metrics path. +# +# ⚠️ NO GATEWAY VHOST, and that omission is deliberate rather than unfinished. +# VictoriaLogs' ingest and query endpoints are unauthenticated, exactly like +# the metrics store's — and the metrics store *is* published under a +# resolvable name, which is an open security question rather than a settled +# design. Publishing this one the same way would repeat that before the first +# instance is decided. The reader is Grafana, which is on this host. +{ + pkgs, + lib, + config, + ... +}: +let + cfg = config.services.hyperhive.swarm.victorialogs; + networkCfg = config.services.hyperhive.network; + hyperhiveCfg = config.services.hyperhive; +in +{ + options.services.hyperhive.swarm.victorialogs = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Run the swarm's log store on this host. Off by default and not + derived from {option}`services.hyperhive.enable`: a swarm has one + log store, so enabling it is a decision about swarm topology + rather than about whether hyperhive is installed. + ''; + }; + + package = lib.mkOption { + type = lib.types.package; + default = pkgs.victorialogs; + defaultText = lib.literalExpression "pkgs.victorialogs"; + description = "VictoriaLogs package to run."; + }; + + machine = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = "swarm-victorialogs"; + description = '' + Container name. Read-only: the name appears in host paths and in + `machinectl`, so it is a fact other modules may read rather than a + knob. + ''; + }; + + port = lib.mkOption { + type = lib.types.port; + default = 9428; + description = '' + Loopback port the store listens on. Upstream's default, kept + because there is no reason to move it and a familiar number is + one less thing an operator has to look up. + + ⚠️ Every swarm container shares the host's network namespace, so + this is a swarm-wide claim rather than a per-container one — two + modules picking the same number collide at runtime with no bind + error and nothing in any log. `state/eval-port-collisions.sh` + checks the class. + ''; + }; + + retentionPeriod = lib.mkOption { + type = lib.types.str; + default = "30d"; + example = "90d"; + description = '' + How long log data is kept. + + Deliberately far shorter than the metrics store's retention: logs + are orders of magnitude larger per unit of time, and their value + decays much faster. A log line answers "what happened during that + incident"; a metric answers "is this worse than last quarter". + ''; + }; + }; + + config = lib.mkIf (hyperhiveCfg.enable && cfg.enable) { + # This store publishes its own health as prometheus metrics on the same + # listener it serves queries on, so the swarm's collector scrapes it with + # no exporter and no extra port — same arrangement as the metrics store. + # + # Declared here rather than in the collector's module because that is the + # rule the option carries: an entry exists only where the service that + # named it runs. ⚠️ That constrains the TARGET, not the scraper — a + # deployment that splits this container away from the collector's host + # silently drops the entry, and no assertion can see it, because separate + # hosts are separate evaluations. + services.hyperhive.swarm.otel.scrapeTargets.victorialogs = "127.0.0.1:${toString cfg.port}"; + + containers.${cfg.machine} = { + autoStart = true; + ephemeral = false; + # Shared host netns, like every sibling swarm container: the collector + # and Grafana reach this at 127.0.0.1:. + privateNetwork = false; + + config = + { ... }: + { + imports = [ + (import ./swarm-container-resolver.nix { + inherit (networkCfg) bridgeIp; + dnsConsumers = [ "victorialogs.service" ]; + }) + ]; + + system.stateVersion = "26.05"; + + # This container shares the host netns, so its own firewall.service + # would rewrite the HOST ruleset at every boot. The host firewall + # owns all filtering. + networking.firewall.enable = false; + # resolvconf stays off because the resolver unit imported above + # owns /etc/resolv.conf. Leaving it on would let host-tracking + # regenerate the file empty, since the host's copy doesn't cross + # the boundary after start. + networking.resolvconf.enable = lib.mkForce false; + + services.victorialogs = { + enable = true; + package = cfg.package; + + # ⚠️ PINNED TO LOOPBACK for the same reason the metrics store is, + # and it matters more here: upstream's default listens on every + # interface, and this endpoint accepts writes as well as reads + # with no authentication of its own. The bind address is the + # boundary. + listenAddress = "127.0.0.1:${toString cfg.port}"; + + extraOptions = [ "-retentionPeriod=${cfg.retentionPeriod}" ]; + }; + }; + }; + }; + + # 🔑 OTLP ingest is served at `/insert/opentelemetry/v1/logs`, measured + # against the pinned build rather than read from docs. The path differs + # from the metrics store's `/opentelemetry/api/v1/push`, so an exporter + # configured by analogy with the metrics one is silently wrong. + # + # ⚠️ And the status code cannot tell you which is which: a wrong path, a + # right path with a wrong body, and a nonsense path ALL answer 400. Only + # the server log distinguishes them — the real route complains about the + # encoding ("json encoding isn't supported ... use protobuf"), while + # anything else logs "unsupported path requested". Verified with a + # nonsense-path control, because a probe where every arm returns the same + # code is not evidence. + # + # ⚠️ Like the metrics store's, that endpoint is unauthenticated — which is + # why `listenAddress` above is loopback, why the collector is the only + # writer, and why this module declares no gateway vhost. +} From 0c755e04c8b9262ee422a126366d90a175e200c5 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 24 Aug 2026 16:48:41 +0200 Subject: [PATCH 2/6] feat(swarm-otel): collect the host journal into the swarm's log store A journald receiver reading the host's journal directory, a logs pipeline stamped with the swarm tier's own resource processor, and an otlphttp exporter pointed at VictoriaLogs. All four parts are conditional on the log store being enabled, so a swarm without one renders exactly as before. The host's directory is enough to see every container: nspawn links a non-ephemeral container's journal guest-side, so the files live on the host under the container's machine-id, and journalctl descends into those subdirectories. Measured, along with the fixed systemd-journal gid that makes the group grant meaningful across the bind mount. --- nix/host-modules/swarm-otel.nix | 94 ++++++++++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 2 deletions(-) diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 4c8cc955..fddaf4c7 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -32,6 +32,7 @@ let swarmCfg = config.services.hyperhive.swarm; otelCfg = config.services.hyperhive.otel; vmCfg = config.services.hyperhive.swarm.victoriametrics; + vlCfg = config.services.hyperhive.swarm.victorialogs; hyperhiveCfg = config.services.hyperhive; gatewayCfg = hyperhiveCfg.gateway; swarmDomain = hyperhiveCfg.swarm.domain; @@ -127,6 +128,18 @@ let # both "the label is absent" and "the label says unknown". swarmDisplayName = if hyperhiveCfg.swarm.name == null then "unknown" else hyperhiveCfg.swarm.name; + # Where the journal lives on BOTH sides of the bind mount below — one string + # because the receiver reads the path it is mounted at, and two spellings of + # one path is a mount that succeeds and a receiver that finds nothing. + # + # 🔑 Why the host's directory is enough to see every container: nspawn is + # invoked with `--link-journal=try-guest` for every non-ephemeral container, + # so a container's journal FILES live here, under its own machine-id + # subdirectory, and are bind-mounted into the guest rather than the other way + # round. Measured: `journalctl -D` on this parent directory descends into + # those subdirectories, so one reader covers the host and every container. + hostJournalDir = "/var/log/journal"; + # One list, read by every pipeline: the per-hive pipelines fan out to # exactly the same destinations as the single pipeline they replace. # Written once because "which exporters" is a property of this tier, not @@ -654,7 +667,17 @@ in } # The public hive CA, read-only — only when something in here # actually verifies a swarm-service name. - // caTrust.bindMount; + // caTrust.bindMount + # The host's journal, read-only, and only when there is a log store to + # write to. Read-only is the whole security posture of this mount: the + # collector has no business writing to a journal, and a collector that + # cannot write cannot corrupt the record it is reporting on. + // lib.optionalAttrs vlCfg.enable { + ${hostJournalDir} = { + hostPath = hostJournalDir; + isReadOnly = true; + }; + }; config = { ... }: @@ -772,6 +795,24 @@ in }; } ) cfg.publishedScrapeTargets; + } + # The whole journal directory, deliberately unfiltered. + # + # A filter restricted to the swarm's own containers was the + # obvious shape and is the wrong one: the host units are where + # the incidents live — the gateway's nginx, the core daemon, + # dnsmasq — and none of them is a swarm container. Filtering to + # `swarm-*` would exclude the single most-needed source. + # + # Nor would a list of container names stay true: it is a second + # place that has to know which services exist, and it goes stale + # silently the next time one is added. Attribution does not need + # it — journald's `_HOSTNAME` / `_SYSTEMD_UNIT` / `_MACHINE_ID` + # are written by journald rather than by the logging process, so + # a reader can tell the origins apart without this collector + # deciding for them. + // lib.optionalAttrs vlCfg.enable { + journald.directory = hostJournalDir; }; exporters = @@ -797,6 +838,22 @@ in headers.${otelCfg.collector.upstreamHeaderName} = "\${env:${otelCfg.collector.upstreamHeaderName}}"; } // lib.optionalAttrs (otelCfg.protocol == "http/json") { encoding = "json"; }; + } + // lib.optionalAttrs vlCfg.enable { + # `logs_endpoint`, NOT `endpoint`, for exactly the reason the + # metrics exporter above spells out — and the trap is worse + # here, because the two stores' OTLP routes differ. `endpoint` + # is a base the exporter appends `/v1/logs` to; VictoriaLogs + # serves `/insert/opentelemetry/v1/logs`, which is also not + # the metrics store's `/opentelemetry/api/v1/push`. Both + # spellings pass `otelcol validate`. + # + # ⚠️ And the store cannot tell you either: a right path, a + # wrong path and a nonsense path all answer 400. Only its log + # distinguishes them — the real route complains about the + # encoding, everything else says "unsupported path requested". + "otlphttp/victorialogs".logs_endpoint = + "http://127.0.0.1:${toString vlCfg.port}/insert/opentelemetry/v1/logs"; }; # Moves this collector's self-metrics off the built-in @@ -850,6 +907,21 @@ in processors = [ "resource/${swarmTierName}" ]; exporters = exporterNames; }; + } + # Logs go to the swarm's store and NOT to `exporterNames`, so + # they do not follow the metrics upstream. That asymmetry is + # deliberate: an operator who configured an upstream endpoint + # agreed to ship metrics there, and quietly adding their logs to + # the same hop is a disclosure decision, not a symmetry fix. + # Same stamp as the scraped swarm services, for the same reason: + # a host's logs belong to the swarm, and there is no honest + # `hive` value to put on them. + // lib.optionalAttrs vlCfg.enable { + "logs/${swarmTierName}" = { + receivers = [ "journald" ]; + processors = [ "resource/${swarmTierName}" ]; + exporters = [ "otlphttp/victorialogs" ]; + }; }; } // { @@ -934,7 +1006,13 @@ in # any one hive, so there is no honest value to put there — and # an invented one (a sentinel, the local hive's name) would be # queried as though it meant something. - // lib.optionalAttrs (cfg.scrapeTargets != { }) { + # + # ⚠️ Emitted for the LOG pipeline too, not just the scraped one: + # a processor a pipeline names but the config does not define is + # a collector that refuses to start, and enabling the log store + # without declaring a scrape target is a perfectly ordinary + # config. + // lib.optionalAttrs (cfg.scrapeTargets != { } || vlCfg.enable) { "resource/${swarmTierName}".attributes = [ { # The metric LABEL, a different namespace from the @@ -966,6 +1044,18 @@ in # scrapes nothing published — the empty case is the shipped one. // lib.optionalAttrs (cfg.publishedScrapeTargets != { }) { LoadCredential = [ "${collectorCredentialId}:${collectorSecretInContainer}" ]; + } + # Journal files are `0640 root:systemd-journal`, and this unit runs + # with upstream's `DynamicUser = true` — so there is no stable uid to + # grant, and the group is the only way in. + # + # 🔑 That works across the bind mount because `systemd-journal` is + # gid **62**, FIXED in nixpkgs' `ids.nix` rather than allocated per + # host. A dynamically allocated gid would mean the host's ownership + # named a different group inside the container, and the failure + # would be a receiver that starts cleanly and reads nothing. + // lib.optionalAttrs vlCfg.enable { + SupplementaryGroups = [ "systemd-journal" ]; }; }; }; From 1af0138928592c5a0e39334c5b16e072a4ee0a22 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 24 Aug 2026 16:59:01 +0200 Subject: [PATCH 3/6] fix(swarm-otel): tell the log store which field carries the message The journald receiver leaves the OTLP body empty and carries the entry as a map of journal fields, so VictoriaLogs had no message to index and wrote a placeholder into _msg on every record. Ingest returned 200, every field was present, and a plain search for a line sitting in the store found nothing. _msg_field names the field that holds the text. _stream_fields is the difference between one stream for the whole host and one per unit per machine; both are set by journald itself and both are low-cardinality. --- nix/host-modules/swarm-otel.nix | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index fddaf4c7..51993a92 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -852,8 +852,26 @@ in # wrong path and a nonsense path all answer 400. Only its log # distinguishes them — the real route complains about the # encoding, everything else says "unsupported path requested". + # 🔑 THE TWO QUERY PARAMETERS ARE NOT TUNING — without the + # first one this pipeline fills the store with records that + # cannot be searched by message text, which is the only + # reason to collect logs at all. + # + # The journald receiver leaves the OTLP *body* empty and + # carries the entry as a map of journal fields, so the store + # has no message to index and writes the literal placeholder + # `missing _msg field` into `_msg` on EVERY record. Nothing + # errors: ingest returns 200, the data is all present, and a + # plain search for a line that is sitting right there returns + # nothing. `_msg_field` tells the store which field carries + # the message. Measured, both with and without. + # + # `_stream_fields` is the difference between one enormous + # stream for the whole host and one per unit per machine — + # both low-cardinality, and both fields journald sets itself. "otlphttp/victorialogs".logs_endpoint = - "http://127.0.0.1:${toString vlCfg.port}/insert/opentelemetry/v1/logs"; + "http://127.0.0.1:${toString vlCfg.port}/insert/opentelemetry/v1/logs" + + "?_msg_field=MESSAGE&_stream_fields=_HOSTNAME,_SYSTEMD_UNIT"; }; # Moves this collector's self-metrics off the built-in From bfa25c6419a9fe7c97533e85d0d9b562abaa92b4 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 24 Aug 2026 17:00:38 +0200 Subject: [PATCH 4/6] feat(swarm): start the log store with the other required services Deferred until the collector had a logs pipeline writing to it: a store nothing writes to starts, answers queries and returns nothing, so the first person to look concludes there were no logs rather than that nothing was collecting them. --- nix/host-modules/swarm-required-services.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nix/host-modules/swarm-required-services.nix b/nix/host-modules/swarm-required-services.nix index 80c3800a..a40b7e56 100644 --- a/nix/host-modules/swarm-required-services.nix +++ b/nix/host-modules/swarm-required-services.nix @@ -71,6 +71,13 @@ in victoriametrics.enable = lib.mkDefault swarmCfg.enableRequiredServices; grafana.enable = lib.mkDefault swarmCfg.enableRequiredServices; + # The log store, deriving from the same switch for the same reason — + # and deliberately in the same commit as the collector pipeline that + # writes to it, never before it. A store nothing writes to is worse + # than no store: it starts, answers queries, and returns nothing, so + # the first person to look concludes there were no logs. + victorialogs.enable = lib.mkDefault swarmCfg.enableRequiredServices; + # The collector that feeds the pair above, and the only tier holding # the upstream credential. Same rule as the rest: once per swarm, # optional, and a hive that is not the service host is a *client* of From f46ef39ef36f2ea5d190ccca03d87575e9d11471 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 24 Aug 2026 17:21:26 +0200 Subject: [PATCH 5/6] feat(swarm-otel): fan logs out like metrics, so the local store is optional The logs half was gated on the swarm's log store being enabled, so turning that store off stopped collection entirely rather than leaving the upstream export. Logs now fan out exactly as metrics do: the store when it runs, the operator's upstream when one is configured, both when both. The receiver, the journal mount and the group grant follow whether there is anywhere to send logs, not whether the local store exists. Tying the mount to the store instead would render a receiver that can read nothing. --- nix/host-modules/swarm-otel.nix | 45 +++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 51993a92..153a6d85 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -140,13 +140,29 @@ let # those subdirectories, so one reader covers the host and every container. hostJournalDir = "/var/log/journal"; + # The operator-configured upstream, named once: the same exporter carries + # every signal, so metrics and logs both reach it without a second + # definition. + upstreamExporterName = if otelCfg.protocol == "grpc" then "otlp" else "otlphttp"; + upstreamExporters = lib.optional (otelCfg.endpoint != "") upstreamExporterName; + # One list, read by every pipeline: the per-hive pipelines fan out to # exactly the same destinations as the single pipeline they replace. # Written once because "which exporters" is a property of this tier, not # of which hive a sample came from. - exporterNames = - lib.optional (otelCfg.endpoint != "") (if otelCfg.protocol == "grpc" then "otlp" else "otlphttp") - ++ lib.optional vmCfg.enable "otlphttp/victoriametrics"; + exporterNames = upstreamExporters ++ lib.optional vmCfg.enable "otlphttp/victoriametrics"; + + # The same fan-out for logs, and the local store is only ONE of its + # destinations. A deployment that turns the swarm's log store off and keeps + # an upstream endpoint still collects — the store is where logs may be kept, + # not the reason to read the journal at all. + logExporterNames = upstreamExporters ++ lib.optional vlCfg.enable "otlphttp/victorialogs"; + + # Collect when there is anywhere to send it, and only then. A pipeline with + # an empty exporter list is not a quiet no-op — the collector rejects it — + # and reading the journal to drop it on the floor would be worse than not + # reading it. + collectLogs = logExporterNames != [ ]; in { options.services.hyperhive.swarm.otel = { @@ -672,7 +688,7 @@ in # write to. Read-only is the whole security posture of this mount: the # collector has no business writing to a journal, and a collector that # cannot write cannot corrupt the record it is reporting on. - // lib.optionalAttrs vlCfg.enable { + // lib.optionalAttrs collectLogs { ${hostJournalDir} = { hostPath = hostJournalDir; isReadOnly = true; @@ -811,7 +827,7 @@ in # are written by journald rather than by the logging process, so # a reader can tell the origins apart without this collector # deciding for them. - // lib.optionalAttrs vlCfg.enable { + // lib.optionalAttrs collectLogs { journald.directory = hostJournalDir; }; @@ -926,19 +942,20 @@ in exporters = exporterNames; }; } - # Logs go to the swarm's store and NOT to `exporterNames`, so - # they do not follow the metrics upstream. That asymmetry is - # deliberate: an operator who configured an upstream endpoint - # agreed to ship metrics there, and quietly adding their logs to - # the same hop is a disclosure decision, not a symmetry fix. + # Logs fan out exactly as metrics do: the swarm's store when it + # runs, the operator's upstream when one is configured, both + # when both. The local store is a destination rather than the + # reason to collect, so turning it off leaves a hive that still + # ships its journal to whoever aggregates across swarms. + # # Same stamp as the scraped swarm services, for the same reason: # a host's logs belong to the swarm, and there is no honest # `hive` value to put on them. - // lib.optionalAttrs vlCfg.enable { + // lib.optionalAttrs collectLogs { "logs/${swarmTierName}" = { receivers = [ "journald" ]; processors = [ "resource/${swarmTierName}" ]; - exporters = [ "otlphttp/victorialogs" ]; + exporters = logExporterNames; }; }; } @@ -1030,7 +1047,7 @@ in # a collector that refuses to start, and enabling the log store # without declaring a scrape target is a perfectly ordinary # config. - // lib.optionalAttrs (cfg.scrapeTargets != { } || vlCfg.enable) { + // lib.optionalAttrs (cfg.scrapeTargets != { } || collectLogs) { "resource/${swarmTierName}".attributes = [ { # The metric LABEL, a different namespace from the @@ -1072,7 +1089,7 @@ in # host. A dynamically allocated gid would mean the host's ownership # named a different group inside the container, and the failure # would be a receiver that starts cleanly and reads nothing. - // lib.optionalAttrs vlCfg.enable { + // lib.optionalAttrs collectLogs { SupplementaryGroups = [ "systemd-journal" ]; }; }; From 012ab5bb37093b28d93059c1f3f3d7786c1520b9 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 24 Aug 2026 17:27:47 +0200 Subject: [PATCH 6/6] fix(swarm-otel): stop re-granting the journal group upstream already grants nixpkgs' collector unit already sets SupplementaryGroups to systemd-journal unconditionally, with a comment saying why. Systemd list options concatenate, so this module's copy rendered ["systemd-journal" "systemd-journal"] and made this a second owner of a fact upstream may later change. The bind mount stays, since that half is genuinely ours. --- nix/host-modules/swarm-otel.nix | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 153a6d85..f87d9c40 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -684,10 +684,23 @@ in # The public hive CA, read-only — only when something in here # actually verifies a swarm-service name. // caTrust.bindMount - # The host's journal, read-only, and only when there is a log store to - # write to. Read-only is the whole security posture of this mount: the - # collector has no business writing to a journal, and a collector that - # cannot write cannot corrupt the record it is reporting on. + # The host's journal, read-only, and only when something will read it. + # Read-only is the whole security posture of this mount: the collector + # has no business writing to a journal, and one that cannot write + # cannot corrupt the record it is reporting on. + # + # The mount is the only half we supply. Journal files are + # `0640 root:systemd-journal` and the unit runs `DynamicUser`, so + # reading them needs that group — which upstream's own collector unit + # already grants unconditionally. Adding it here again is not + # harmless: systemd list options CONCATENATE, so a second copy renders + # `[ "systemd-journal" "systemd-journal" ]` and quietly becomes a + # second owner of a fact upstream may later change. + # + # It works across the mount because that gid is FIXED at 62 in + # nixpkgs' `ids.nix`. A per-host allocation would leave the host's + # ownership naming a different group inside the container, and the + # failure would be a receiver that starts cleanly and reads nothing. // lib.optionalAttrs collectLogs { ${hostJournalDir} = { hostPath = hostJournalDir; @@ -1079,18 +1092,6 @@ in # scrapes nothing published — the empty case is the shipped one. // lib.optionalAttrs (cfg.publishedScrapeTargets != { }) { LoadCredential = [ "${collectorCredentialId}:${collectorSecretInContainer}" ]; - } - # Journal files are `0640 root:systemd-journal`, and this unit runs - # with upstream's `DynamicUser = true` — so there is no stable uid to - # grant, and the group is the only way in. - # - # 🔑 That works across the bind mount because `systemd-journal` is - # gid **62**, FIXED in nixpkgs' `ids.nix` rather than allocated per - # host. A dynamically allocated gid would mean the host's ownership - # named a different group inside the container, and the failure - # would be a receiver that starts cleanly and reads nothing. - // lib.optionalAttrs collectLogs { - SupplementaryGroups = [ "systemd-journal" ]; }; }; };