vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-otel ships its own journal, so its logs reach the store

Browse source 
The collector's container ran with --link-journal=host and the host's
journal bound over /var/log/journal, so its own journal was never
written where anything read it. It now keeps its journal in the guest,
reads it with a second journald receiver, and gets the host's journal
at /run/host-journal.

On deploy swarm-otel rows start arriving in the store, stamped
service.name swarm-otel.
This commit is contained in:
atlas 2026-10-19 20:25:23 +02:00
commit 8476609f98
3 changed files with 83 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

74
nix/host-modules/swarm-otel.nix
View file

@ -197,29 +197,20 @@ let
# both "the label is absent" and "the label says unknown".
swarmDisplayName = if hyperhiveCfg.swarm.name == null then "unknown" else hyperhiveCfg.swarm.name;
# Where the journal lives on BOTH sides of the bind mount below — one string
# because the receiver reads the path it is mounted at, and two spellings of
# one path is a mount that succeeds and a receiver that finds nothing.
# The host's journal directory, and where it is bind-mounted in the guest.
# The mount point is NOT the guest's own /var/log/journal: that is where this
# container's journald writes and `journald/self` reads, and a mount over it
# hides the guest's journal from its own reader.
#
# 🔑 Why the host's directory is enough to see a container — and why it is
# only enough because that container asks for it. nixpkgs hardcodes
# `--link-journal=try-guest`, which puts the journal inside the container and
# leaves the host with a symlink into that container's transient root: a
# reader here cannot follow it, and it dangles the moment the container
# stops. Every container block except the `serviceProducers` ones therefore sets
# `extraFlags = [ "--link-journal=host" ]`, which the invocation expands
# AFTER the hardcoded flag, so the files land here under their own
# machine-id subdirectory and are bind-mounted into the guest instead.
#
# ⚠️ Drop that flag from one of THOSE containers and this receiver silently
# stops seeing it — no error, just a unit that never appears in the store.
# The `serviceProducers` containers (`swarm-bao`, `swarm-nats`,
# nixpkgs hardcodes `--link-journal=try-guest`, which keeps this container's
# journal in the guest, so the host directory holds the host's units only.
# Every container with a `serviceProducers` entry (`swarm-bao`, `swarm-nats`,
# `swarm-builder`, `swarm-claude-login`, `swarm-review-classifier`,
# `swarm-ci-runner`, `swarm-grafana`, `swarm-victorialogs`, `hive-forge`,
# `hive-matrix`, `swarm-victoriametrics`, `swarm-authelia`) are the exceptions
# on purpose: their journals stay inside the container for its own collector
# instead, so this receiver never sees their units.
# `hive-matrix`, `swarm-victoriametrics`, `swarm-authelia`) forwards its own
# journal from its own collector, so this receiver never sees their units.
hostJournalDir = "/var/log/journal";
hostJournalMount = "/run/host-journal";
# Each store's OTLP route, by domain. One binding because the same string is
# both the address requested and the audience the token is minted for — two
@ -802,9 +793,8 @@ in
containers.${cfg.machine} = {
autoStart = true;
ephemeral = false;
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [ "--link-journal=host" ];
# No `--link-journal=host`: this container's journal stays in the guest,
# where `journald/self` reads it.
inherit privateNetwork;
# The upstream credential is operator-provided and lives on the host.
@ -846,7 +836,7 @@ in
# ownership naming a different group inside the container, and the
# failure would be a receiver that starts cleanly and reads nothing.
// lib.optionalAttrs collectLogs {
${hostJournalDir} = {
${hostJournalMount} = {
hostPath = hostJournalDir;
isReadOnly = true;
};
@ -1021,7 +1011,7 @@ in
# machine id is per-machine by construction.
// lib.optionalAttrs collectLogs {
journald = {
directory = hostJournalDir;
directory = hostJournalMount;
# The SAME operator list the agent tier attaches to its own
# receiver (nix/agent-modules/otel.nix), and it transfers
# without adjustment: the entry shape is the receiver's,
@ -1047,6 +1037,21 @@ in
# the journal it indexes does.
storage = "file_storage";
};
# This container's own journal, read in the guest. The
# collector's export failures land here and ship like any
# other record; the read cursor is the receiver's own, so
# a collector that is down at boot loses nothing and waits
# on nothing.
#
# ⚠️ `merge` is stated: without `--merge`, `journalctl
# --follow` over a journal with no entry for the current
# boot crash-loops the receiver.
"journald/self" = {
directory = "/var/log/journal";
merge = true;
operators = import ../journald-severity.nix;
storage = "file_storage";
};
};
exporters = {
@ -1240,6 +1245,17 @@ in
];
exporters = logExporterNames;
};
# This container's own journal goes straight to the stores,
# not through `otlp/<name>`: a collector forwarding to
# itself would need its own receiver up first.
"logs/self" = {
receivers = [ "journald/self" ];
processors = [
"resource/self-journal"
"resource/${swarmTierName}"
];
exporters = logExporterNames;
};
};
}
// {
@ -1426,6 +1442,16 @@ in
# `ignore`: a record whose body is not a map cannot be indexed,
# and is kept (with a warning) rather than dropped.
// lib.optionalAttrs collectLogs {
# `service.name` on this container's own journal, as the
# other containers' forwarders stamp theirs.
"resource/self-journal".attributes = [
{
key = "service.name";
value = cfg.machine;
action = "upsert";
}
];
"filter/exclude-user-sessions" = {
error_mode = "ignore";
logs.log_record = [

26
nix/module-eval/swarm-otel-core.nix
View file

@ -259,6 +259,32 @@ let
# The window has to outlast every attempt, or the burst is unreachable.
&& u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst;
}
{
# 🩸 The container's own journal reaches the stores from the collector
# itself. Mounting the host's journal over the guest's /var/log/journal
# (or `--link-journal=host`) leaves `journald/self` reading nothing while
# the collector looks healthy. Presence controls: the host receiver and
# its mount are asserted too, so a misspelt path cannot pass.
name = "the swarm collector ships its own journal, and the host's from a mount that leaves it alone";
ok =
let
c = otelNoStores.containers.swarm-otel;
s = otelSettings otelNoStores;
self = lib.filter (p: builtins.elem "journald/self" p.receivers) (
lib.attrValues s.service.pipelines
);
in
!(lib.elem "--link-journal=host" (c.extraFlags or [ ]))
&& s.receivers."journald/self".directory == "/var/log/journal"
&& s.receivers."journald/self".merge
&& s.receivers.journald.directory == "/run/host-journal"
&& c.bindMounts."/run/host-journal".hostPath == "/var/log/journal"
&& !(c.bindMounts ? "/var/log/journal")
&& lib.length self == 1
&& lib.all (p: p.exporters == [ "otlphttp/victorialogs" ]) self
&& lib.elem "resource/self-journal" (lib.head self).processors
&& builtins.elem "resource/self-journal" (lib.attrNames s.processors);
}
];
in
runGroup "swarm-otel-core" cases