diff --git a/docs/swarm/services.md b/docs/swarm/services.md index f239e148..791f2c94 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -348,6 +348,13 @@ authelia, forge and homeserver have a forwarder client of their own `tuwunel`) and no `hive` label. The forwarder of all but the store's is `nix/container-modules/swarm-journal-forwarder.nix`. +The swarm collector's own container, `swarm-otel`, has no forwarder: its +collector reads the container's journal itself (receiver `journald/self`) and +exports it to the log store like the host's journal, stamped `service.name` +`swarm-otel` and with no `hive` label. The container mounts the host's journal +read-only at `/run/host-journal`, not over its own +`/var/log/journal`. + **Both store exporters are unconditional**, and `deploy.victoriametrics.enable` doesn't gate them: that option says this host _runs_ the store, while the swarm has one either way, reached by its swarm name through the gateway. diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 95d1b0ca..ce3ef972 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -197,29 +197,20 @@ let # both "the label is absent" and "the label says unknown". swarmDisplayName = if hyperhiveCfg.swarm.name == null then "unknown" else hyperhiveCfg.swarm.name; - # Where the journal lives on BOTH sides of the bind mount below — one string - # because the receiver reads the path it is mounted at, and two spellings of - # one path is a mount that succeeds and a receiver that finds nothing. + # The host's journal directory, and where it is bind-mounted in the guest. + # The mount point is NOT the guest's own /var/log/journal: that is where this + # container's journald writes and `journald/self` reads, and a mount over it + # hides the guest's journal from its own reader. # - # 🔑 Why the host's directory is enough to see a container — and why it is - # only enough because that container asks for it. nixpkgs hardcodes - # `--link-journal=try-guest`, which puts the journal inside the container and - # leaves the host with a symlink into that container's transient root: a - # reader here cannot follow it, and it dangles the moment the container - # stops. Every container block except the `serviceProducers` ones therefore sets - # `extraFlags = [ "--link-journal=host" ]`, which the invocation expands - # AFTER the hardcoded flag, so the files land here under their own - # machine-id subdirectory and are bind-mounted into the guest instead. - # - # ⚠️ Drop that flag from one of THOSE containers and this receiver silently - # stops seeing it — no error, just a unit that never appears in the store. - # The `serviceProducers` containers (`swarm-bao`, `swarm-nats`, + # nixpkgs hardcodes `--link-journal=try-guest`, which keeps this container's + # journal in the guest, so the host directory holds the host's units only. + # Every container with a `serviceProducers` entry (`swarm-bao`, `swarm-nats`, # `swarm-builder`, `swarm-claude-login`, `swarm-review-classifier`, # `swarm-ci-runner`, `swarm-grafana`, `swarm-victorialogs`, `hive-forge`, - # `hive-matrix`, `swarm-victoriametrics`, `swarm-authelia`) are the exceptions - # on purpose: their journals stay inside the container for its own collector - # instead, so this receiver never sees their units. + # `hive-matrix`, `swarm-victoriametrics`, `swarm-authelia`) forwards its own + # journal from its own collector, so this receiver never sees their units. hostJournalDir = "/var/log/journal"; + hostJournalMount = "/run/host-journal"; # Each store's OTLP route, by domain. One binding because the same string is # both the address requested and the audience the token is minted for — two @@ -802,9 +793,8 @@ in containers.${cfg.machine} = { autoStart = true; ephemeral = false; - # Journal files on the host, not inside the container: nixpkgs hardcodes - # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it. - extraFlags = [ "--link-journal=host" ]; + # No `--link-journal=host`: this container's journal stays in the guest, + # where `journald/self` reads it. inherit privateNetwork; # The upstream credential is operator-provided and lives on the host. @@ -846,7 +836,7 @@ in # ownership naming a different group inside the container, and the # failure would be a receiver that starts cleanly and reads nothing. // lib.optionalAttrs collectLogs { - ${hostJournalDir} = { + ${hostJournalMount} = { hostPath = hostJournalDir; isReadOnly = true; }; @@ -1021,7 +1011,7 @@ in # machine id is per-machine by construction. // lib.optionalAttrs collectLogs { journald = { - directory = hostJournalDir; + directory = hostJournalMount; # The SAME operator list the agent tier attaches to its own # receiver (nix/agent-modules/otel.nix), and it transfers # without adjustment: the entry shape is the receiver's, @@ -1047,6 +1037,21 @@ in # the journal it indexes does. storage = "file_storage"; }; + # This container's own journal, read in the guest. The + # collector's export failures land here and ship like any + # other record; the read cursor is the receiver's own, so + # a collector that is down at boot loses nothing and waits + # on nothing. + # + # ⚠️ `merge` is stated: without `--merge`, `journalctl + # --follow` over a journal with no entry for the current + # boot crash-loops the receiver. + "journald/self" = { + directory = "/var/log/journal"; + merge = true; + operators = import ../journald-severity.nix; + storage = "file_storage"; + }; }; exporters = { @@ -1240,6 +1245,17 @@ in ]; exporters = logExporterNames; }; + # This container's own journal goes straight to the stores, + # not through `otlp/`: a collector forwarding to + # itself would need its own receiver up first. + "logs/self" = { + receivers = [ "journald/self" ]; + processors = [ + "resource/self-journal" + "resource/${swarmTierName}" + ]; + exporters = logExporterNames; + }; }; } // { @@ -1426,6 +1442,16 @@ in # `ignore`: a record whose body is not a map cannot be indexed, # and is kept (with a warning) rather than dropped. // lib.optionalAttrs collectLogs { + # `service.name` on this container's own journal, as the + # other containers' forwarders stamp theirs. + "resource/self-journal".attributes = [ + { + key = "service.name"; + value = cfg.machine; + action = "upsert"; + } + ]; + "filter/exclude-user-sessions" = { error_mode = "ignore"; logs.log_record = [ diff --git a/nix/module-eval/swarm-otel-core.nix b/nix/module-eval/swarm-otel-core.nix index 34a04165..3b17f943 100644 --- a/nix/module-eval/swarm-otel-core.nix +++ b/nix/module-eval/swarm-otel-core.nix @@ -259,6 +259,32 @@ let # The window has to outlast every attempt, or the burst is unreachable. && u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst; } + { + # 🩸 The container's own journal reaches the stores from the collector + # itself. Mounting the host's journal over the guest's /var/log/journal + # (or `--link-journal=host`) leaves `journald/self` reading nothing while + # the collector looks healthy. Presence controls: the host receiver and + # its mount are asserted too, so a misspelt path cannot pass. + name = "the swarm collector ships its own journal, and the host's from a mount that leaves it alone"; + ok = + let + c = otelNoStores.containers.swarm-otel; + s = otelSettings otelNoStores; + self = lib.filter (p: builtins.elem "journald/self" p.receivers) ( + lib.attrValues s.service.pipelines + ); + in + !(lib.elem "--link-journal=host" (c.extraFlags or [ ])) + && s.receivers."journald/self".directory == "/var/log/journal" + && s.receivers."journald/self".merge + && s.receivers.journald.directory == "/run/host-journal" + && c.bindMounts."/run/host-journal".hostPath == "/var/log/journal" + && !(c.bindMounts ? "/var/log/journal") + && lib.length self == 1 + && lib.all (p: p.exporters == [ "otlphttp/victorialogs" ]) self + && lib.elem "resource/self-journal" (lib.head self).processors + && builtins.elem "resource/self-journal" (lib.attrNames s.processors); + } ]; in runGroup "swarm-otel-core" cases