From 9c601c416685de28383dd447e3a12adec7876e64 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 2 Sep 2026 23:00:30 +0200 Subject: [PATCH] containers: put journal files on the host, where the collector looks The swarm collector reads /var/log/journal and has never seen a single container unit. mara's count-by-unit against VictoriaLogs returns four units, all host-tier; openbao -- which runs inside the swarm-bao container -- is absent. Cause: nixos-containers.nix hardcodes `--link-journal=try-guest` for every non-ephemeral container. With `guest`, the host's /var/log/journal/ is a SYMLINK into the container's transient root; a reader in the host namespace cannot follow it, and it dangles as soon as the container stops. `ls -la /var/log/journal/` on the host shows one real directory and a pile of `-> /tmp/nspawn-root-*` links dating back to May. swarm-otel.nix asserted the opposite, and that assertion is why the receiver's path was considered sufficient: it said the files "live here" and are "bind-mounted into the guest rather than the other way round". That describes `--link-journal=host`. The same sentence names the flag we actually use. The flag was right and the behaviour it described was not, so grepping for the flag confirmed the comment and taught nothing. `containers..extraFlags` feeds EXTRA_NSPAWN_FLAGS, which the invocation expands after the hardcoded flag, so `--link-journal=host` wins. The comment now describes what the code does instead of the other way round. Two payoffs, and the smaller one is the one the issue is about: container logs become collectable, and -- independently -- they become durable at all, rather than dying with the container. Ten identical edits because ten host-modules hand-roll their own container block; that duplication is #3773, not something to invent an abstraction for here. NOT VERIFIED: that systemd-nspawn honours the last `--link-journal` of two. Everything else here is read out of nixpkgs; that step is a claim about its argument parsing which cannot be exercised without starting a container. It is settled by deploying one and re-running the `ls`: the machine-id entry becomes a real directory instead of a symlink. Refs #3849 --- nix/host-modules/hive-ci.nix | 3 +++ nix/host-modules/hive-forge/default.nix | 3 +++ nix/host-modules/hive-matrix.nix | 3 +++ nix/host-modules/swarm-authelia.nix | 3 +++ nix/host-modules/swarm-bao.nix | 3 +++ nix/host-modules/swarm-grafana.nix | 3 +++ nix/host-modules/swarm-nats.nix | 3 +++ nix/host-modules/swarm-otel.nix | 21 +++++++++++++++------ nix/host-modules/swarm-victorialogs.nix | 3 +++ nix/host-modules/swarm-victoriametrics.nix | 3 +++ 10 files changed, 42 insertions(+), 6 deletions(-) diff --git a/nix/host-modules/hive-ci.nix b/nix/host-modules/hive-ci.nix index adede597..e52f8adb 100644 --- a/nix/host-modules/hive-ci.nix +++ b/nix/host-modules/hive-ci.nix @@ -210,6 +210,9 @@ in containers.hive-ci = { autoStart = true; ephemeral = false; + # Journal files on the host, not inside the container: nixpkgs hardcodes + # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it. + extraFlags = [ "--link-journal=host" ]; # Private network namespace, attached to the hive bridge so the # runner reaches the forge via the gateway — and cannot reach # host-loopback (127.0.0.1:7000 dashboard, raw forge port, etc.). diff --git a/nix/host-modules/hive-forge/default.nix b/nix/host-modules/hive-forge/default.nix index 0a5f4dc2..43bc9ea1 100644 --- a/nix/host-modules/hive-forge/default.nix +++ b/nix/host-modules/hive-forge/default.nix @@ -634,6 +634,9 @@ in containers.hive-forge = { autoStart = true; ephemeral = false; + # Journal files on the host, not inside the container: nixpkgs hardcodes + # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it. + extraFlags = [ "--link-journal=host" ]; # Share host netns — forgejo's HTTP / SSH listeners then look # exactly like a host-side service, no port forwarding dance, # and agent containers (which also share host netns) reach it diff --git a/nix/host-modules/hive-matrix.nix b/nix/host-modules/hive-matrix.nix index 5099973b..cdeaf9ef 100644 --- a/nix/host-modules/hive-matrix.nix +++ b/nix/host-modules/hive-matrix.nix @@ -825,6 +825,9 @@ in containers.hive-matrix = { autoStart = true; ephemeral = false; + # Journal files on the host, not inside the container: nixpkgs hardcodes + # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it. + extraFlags = [ "--link-journal=host" ]; # Shared host netns — agents reach tuwunel at localhost:. privateNetwork = false; # Read-only bind of the host-managed registration token; tuwunel diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index ea2edf7c..9684293f 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -1070,6 +1070,9 @@ in containers.${cfg.machine} = { autoStart = true; ephemeral = false; + # Journal files on the host, not inside the container: nixpkgs hardcodes + # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it. + extraFlags = [ "--link-journal=host" ]; # Shared host netns, like the forge and matrix containers: the # gateway reaches authelia at 127.0.0.1:. privateNetwork = false; diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index b43ab84e..6ba6e96f 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -690,6 +690,9 @@ in containers.${cfg.machine} = { autoStart = true; ephemeral = false; + # Journal files on the host, not inside the container: nixpkgs hardcodes + # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it. + extraFlags = [ "--link-journal=host" ]; # Shared host netns, like every sibling swarm container. Unlike them the # gateway is NOT the client here (see the no-vhost note at the top), so # sharing the netns is what lets the store bind the host's own addresses diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 26c0d8ae..5c083c86 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -497,6 +497,9 @@ in containers.${cfg.machine} = { autoStart = true; ephemeral = false; + # Journal files on the host, not inside the container: nixpkgs hardcodes + # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it. + extraFlags = [ "--link-journal=host" ]; # Shared host netns, like every sibling swarm container: the gateway # reaches this at 127.0.0.1:. privateNetwork = false; diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index d4c68b12..989522a6 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -544,6 +544,9 @@ in containers.swarm-nats = { autoStart = true; ephemeral = false; + # Journal files on the host, not inside the container: nixpkgs hardcodes + # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it. + extraFlags = [ "--link-journal=host" ]; # Shared host netns, like every sibling container. # # ⚠️ Which is exactly why the server below must refuse everyone diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 992811b7..a6e8f278 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -179,12 +179,18 @@ let # because the receiver reads the path it is mounted at, and two spellings of # one path is a mount that succeeds and a receiver that finds nothing. # - # 🔑 Why the host's directory is enough to see every container: nspawn is - # invoked with `--link-journal=try-guest` for every non-ephemeral container, - # so a container's journal FILES live here, under its own machine-id - # subdirectory, and are bind-mounted into the guest rather than the other way - # round. `journalctl -D` on this parent directory descends into - # those subdirectories, so one reader covers the host and every container. + # 🔑 Why the host's directory is enough to see every container — and why it + # is only enough because each container asks for it. nixpkgs hardcodes + # `--link-journal=try-guest`, which puts the journal inside the container and + # leaves the host with a symlink into that container's transient root: a + # reader here cannot follow it, and it dangles the moment the container + # stops. Every container block therefore sets + # `extraFlags = [ "--link-journal=host" ]`, which the invocation expands + # AFTER the hardcoded flag, so the files land here under their own + # machine-id subdirectory and are bind-mounted into the guest instead. + # + # ⚠️ Drop that flag from a container and this receiver silently stops seeing + # it — no error, just a unit that never appears in the store. hostJournalDir = "/var/log/journal"; # Each store's OTLP route, by domain. One binding because the same string is @@ -911,6 +917,9 @@ in containers.${cfg.machine} = { autoStart = true; ephemeral = false; + # Journal files on the host, not inside the container: nixpkgs hardcodes + # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it. + extraFlags = [ "--link-journal=host" ]; # Shared host netns, like every sibling swarm service: the hive tier # reaches this collector, and this collector reaches the metrics # store, without either crossing a network boundary that would need diff --git a/nix/host-modules/swarm-victorialogs.nix b/nix/host-modules/swarm-victorialogs.nix index b9d8c334..4709e3eb 100644 --- a/nix/host-modules/swarm-victorialogs.nix +++ b/nix/host-modules/swarm-victorialogs.nix @@ -228,6 +228,9 @@ in containers.${cfg.machine} = { autoStart = true; ephemeral = false; + # Journal files on the host, not inside the container: nixpkgs hardcodes + # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it. + extraFlags = [ "--link-journal=host" ]; # Shared host netns, like every sibling swarm container: the collector # and Grafana reach this at 127.0.0.1:. privateNetwork = false; diff --git a/nix/host-modules/swarm-victoriametrics.nix b/nix/host-modules/swarm-victoriametrics.nix index 31f24ade..7e080d97 100644 --- a/nix/host-modules/swarm-victoriametrics.nix +++ b/nix/host-modules/swarm-victoriametrics.nix @@ -182,6 +182,9 @@ in containers.${cfg.machine} = { autoStart = true; ephemeral = false; + # Journal files on the host, not inside the container: nixpkgs hardcodes + # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it. + extraFlags = [ "--link-journal=host" ]; # Shared host netns, like every sibling swarm container: the gateway # reaches this at 127.0.0.1:. privateNetwork = false;