From 44009dc51d6d26b274134c1e086eaf5d7be9e987 Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 24 Sep 2026 23:12:10 +0200 Subject: [PATCH 1/2] swarm-bao: stop linking the container's journal onto the host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --link-journal=host bind-mounts a host directory journald never writes into for this container (empty, root:nogroup, confirmed on the live host — #4527). Dropping it falls back to nixpkgs' default --link-journal=try-guest, the same shape every agent container already uses, so the in-container collector's journald receiver (directory = /var/log/journal) now reads a journal that is actually written. merge stays true and services.hyperhive.swarm.otel.journaldUnits is untouched so this deploy changes exactly one thing; comments that described the old host-linked shape are rewritten to match. Adds a module-eval assertion (bao-otel-collector.nix) that the container's extraFlags never re-add --link-journal=host. Refs #4527, #4499. --- nix/host-modules/swarm-bao.nix | 47 ++++++++++++++------------ nix/module-eval/bao-otel-collector.nix | 10 ++++++ 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index bde146cb..d10996a4 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -1645,9 +1645,13 @@ in # collector (see `receivers.prometheus` below), which travels with the # store. # - # Logs are untouched by that move: `journaldUnits` above still names this - # store's units for the shared collector, and the two are retired - # together once every sibling swarm container has a collector of its own. + # Logs are a different story from that move: `journaldUnits` above still + # names this store's units, but bao's journal now lives only inside the + # container (see the forwarder's own comment below) — nothing links it + # into the host tree any more, so the shared collector can never see + # these units. The list stays untouched for the sibling containers that + # still ride it; bao's own entries come out once delivery through the + # container's own collector is confirmed. # ⚠️ The one nginx exception to this file's header, and it is one because # it never terminates. `ssl_preread` reads the SNI off the ClientHello @@ -2409,9 +2413,6 @@ in containers.${cfg.machine} = { autoStart = true; ephemeral = false; - # Journal files on the host, not inside the container: nixpkgs hardcodes - # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it. - extraFlags = [ "--link-journal=host" ]; # Shared host netns, like every sibling swarm container. Unlike them the # gateway is NOT the client here (see the no-vhost note at the top), so # sharing the netns is what lets the store bind the host's own addresses @@ -2660,12 +2661,11 @@ in # is supposed to ship its own logs to the next hop rather than # leave a *different* container's collector to find them. # - # That older shape is what `--link-journal=host` above and the - # `swarm.otel.journaldUnits` entry near the top of this file serve: - # the journal lands in the host tree, where the swarm collector — - # itself a container — reads it through a unit allowlist. Both stay - # for now, because every sibling swarm container still rides that - # collector; they come out once each of them has one of these. + # The journal lives inside this container now, the same shape as + # every agent container: nothing links it into the host's + # /var/log/journal tree, so `swarm.otel.journaldUnits` near the + # top of this file can no longer reach any of bao's units through + # the shared collector — see the comment there. # # ⚠️ NO `units` allowlist here, and that is the point rather than a # simplification. A shared collector needs one because the journal @@ -2730,17 +2730,20 @@ in receivers.journald = { # ⚠️ STATED, and it must stay stated: the receiver's own - # default is the RUNTIME journal (`/run/log/journal`), and - # every entry here is under /var/log/journal — which - # `--link-journal=host` above makes the host's directory for - # this machine id. Dropping this line leaves a collector that - # validates, starts, reports healthy and forwards nothing. + # default is the RUNTIME journal (`/run/log/journal`), which + # in this container is empty — journald stores persistently + # here, so every entry is under /var/log/journal. Dropping + # this line leaves a collector that validates, starts, + # reports healthy and forwards nothing. directory = "/var/log/journal"; - # ⚠️ `--merge`, and it is what makes this receiver read - # anything at all: `journalctl --follow` scopes itself to the - # current boot, which has no entry under the host's journal - # tree — so every start died with "No journal boot entry found - # for the specified boot (+0)" in a ~2s crash-loop. + # ⚠️ `--merge` stays from when this journal was host-linked + # and `journalctl --follow` (unqualified) found no entry for + # the current boot under that tree, crash-looping the + # receiver every ~2s with "No journal boot entry found for + # the specified boot (+0)". The journal is local now, so the + # current boot has an entry here too — kept rather than + # dropped so the first deploy changes exactly one thing; + # revisit once delivery is confirmed. merge = true; storage = "file_storage"; }; diff --git a/nix/module-eval/bao-otel-collector.nix b/nix/module-eval/bao-otel-collector.nix index d852f5de..af38f960 100644 --- a/nix/module-eval/bao-otel-collector.nix +++ b/nix/module-eval/bao-otel-collector.nix @@ -265,6 +265,16 @@ let in (j.merge or false) == true && j.directory == "/var/log/journal"; } + { + # The empty-directory trap this receiver used to fall into: `host` mode + # bind-mounts a host directory journald never actually writes into, so + # `merge`/`directory` above would keep validating and starting while + # the receiver reads nothing — silently, same as every other failure + # mode here. Guards against re-adding the exact flag this was fixed by + # dropping. + name = "the store's container does not link its journal onto the host"; + ok = !(lib.elem "--link-journal=host" (baoWithCollector.containers.swarm-bao.extraFlags or [ ])); + } { # The whole journal, which is what the shared collector's unit allowlist # is not. A `units` list here would render and deploy perfectly while From c4edb78aa27f9cd5083bfea19662a9a9ad025355 Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 24 Sep 2026 23:44:32 +0200 Subject: [PATCH 2/2] swarm-otel: correct the hostJournalDir comment for swarm-bao's exception argus review on #4713: the comment said every container block sets --link-journal=host, and warned that dropping it silently blinds this receiver. #4713 does exactly that for swarm-bao (its journal stays inside the container for its own collector instead), so read on its own the comment told a debugger to revert the fix. State the exception. --- nix/host-modules/swarm-otel.nix | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index abf2231c..edb36ff5 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -202,18 +202,21 @@ let # because the receiver reads the path it is mounted at, and two spellings of # one path is a mount that succeeds and a receiver that finds nothing. # - # 🔑 Why the host's directory is enough to see every container — and why it - # is only enough because each container asks for it. nixpkgs hardcodes + # 🔑 Why the host's directory is enough to see a container — and why it is + # only enough because that container asks for it. nixpkgs hardcodes # `--link-journal=try-guest`, which puts the journal inside the container and # leaves the host with a symlink into that container's transient root: a # reader here cannot follow it, and it dangles the moment the container - # stops. Every container block therefore sets + # stops. Every container block except `swarm-bao` therefore sets # `extraFlags = [ "--link-journal=host" ]`, which the invocation expands # AFTER the hardcoded flag, so the files land here under their own # machine-id subdirectory and are bind-mounted into the guest instead. # - # ⚠️ Drop that flag from a container and this receiver silently stops seeing - # it — no error, just a unit that never appears in the store. + # ⚠️ Drop that flag from one of THOSE containers and this receiver silently + # stops seeing it — no error, just a unit that never appears in the store. + # `swarm-bao` is the one exception on purpose: its journal stays inside the + # container for its own collector instead (see `swarm-bao.nix`), so it was + # never one of the units this receiver could see either. hostJournalDir = "/var/log/journal"; # Each store's OTLP route, by domain. One binding because the same string is