Watch
0
0
Fork
You've already forked hyperhive
0

swarm-otel: collect only the units the swarm's services declare

The journald receiver was configured with a directory and no filter, so
the swarm's log store received every unit on the host that runs the
collector. On a hive whose services live on a workstation that includes
the operator's desktop session, in a store every swarm operator can read.

The receiver has no system-only switch and its `matches` field is an
allowlist too, so what the swarm collects has to be stated rather than
excluded. Each service module names its own units: a service that is not
running contributes nothing, and one added later arrives declared.

An empty list is fail-open — the receiver renders no filter at all and
reads everything — so it is asserted against.
This commit is contained in:
atlas 2026-08-24 21:54:45 +02:00 • committed by mara
commit 4336436457
11 changed files with 145 additions and 14 deletions

View file

@ -136,6 +136,14 @@ in
]; ];
config = lib.mkIf cfg.enable { config = lib.mkIf cfg.enable {
# The daemon that owns every container on this hive, and the helper it
# delegates its root operations to. An agent asking why a container did
# not come up is asking about one of these two.
services.hyperhive.swarm.otel.journaldUnits = [
"hive-c0re"
"hive-priv"
];
assertions = [ assertions = [
{ {
# The pinned claude reaches agents as a bare path, so this # The pinned claude reaches agents as a bare path, so this

View file

@ -420,6 +420,14 @@ in
}; };
config = lib.mkIf config.services.hyperhive.enable { config = lib.mkIf config.services.hyperhive.enable {
# Same principle as the vhost below — this service's own surface lives
# with the service. The SSO source is named alongside forgejo because
# its failure mode is a login that silently falls back, not an error.
services.hyperhive.swarm.otel.journaldUnits = [
"forgejo"
"forgejo-sso-source"
];
# This service's own gateway surface: the vhost that fronts it and # This service's own gateway surface: the vhost that fronts it and
# the name the hive resolver answers for. Declared here rather than # the name the hive resolver answers for. Declared here rather than
# in the gateway so the forge's public face lives with the forge — # in the gateway so the forge's public face lives with the forge —

View file

@ -115,6 +115,18 @@ in
# file's `let` and are not option surface. # file's `let` and are not option surface.
services.hyperhive.gateway.lib = vhostLib; services.hyperhive.gateway.lib = vhostLib;
# Every request to every hyperhive service passes through here, so this
# is the one unit that can say a service was unreachable rather than
# merely quiet. Named even on hives that run no swarm collector: the
# option is inert unless one is collecting on this host.
#
# dnsmasq alongside it for the same reason one level down: a name that
# stops resolving presents as every client timing out at once.
services.hyperhive.swarm.otel.journaldUnits = [
"nginx"
"dnsmasq"
];
assertions = [ assertions = [
{ {
assertion = !(cfg.tls.acme.enable && cfg.tls.certDir != null); assertion = !(cfg.tls.acme.enable && cfg.tls.certDir != null);

View file

@ -983,6 +983,13 @@ in
# bridge at that wrong answer. # bridge at that wrong answer.
services.hyperhive.gateway.localNames = [ cfg.domain ]; services.hyperhive.gateway.localNames = [ cfg.domain ];
# The bridge as well as authelia: it is the half that writes the identity
# store, and its refusals are returned to callers as a bare 401.
services.hyperhive.swarm.otel.journaldUnits = [
"authelia"
"swarm-authelia-bridge"
];
# Declared here rather than in the collector's module, per the option's # Declared here rather than in the collector's module, per the option's
# own rule: an entry exists only where the service that named it runs. # own rule: an entry exists only where the service that named it runs.
# #

View file

@ -138,6 +138,13 @@ in
}; };
config = lib.mkIf (hyperhiveCfg.enable && cfg.autoConfigure) { config = lib.mkIf (hyperhiveCfg.enable && cfg.autoConfigure) {
# A CA that fails to issue is invisible until something makes a TLS call
# hours later, so these two oneshots are worth more than most services.
services.hyperhive.swarm.otel.journaldUnits = [
"swarm-ca"
"swarm-services-ca"
];
systemd.services.swarm-ca = { systemd.services.swarm-ca = {
description = "Generate the swarm root CA when absent"; description = "Generate the swarm root CA when absent";
wantedBy = [ "multi-user.target" ]; wantedBy = [ "multi-user.target" ];

View file

@ -443,6 +443,13 @@ in
}; };
config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) { config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) {
# The daemon and the oneshot that mints its credential — the second one
# failing leaves the first running and unable to authenticate anywhere.
services.hyperhive.swarm.otel.journaldUnits = [
"swarm-controller"
"swarm-controller-credential"
];
users.users.swarm-controller = { users.users.swarm-controller = {
isSystemUser = true; isSystemUser = true;
group = "swarm-controller"; group = "swarm-controller";

View file

@ -315,6 +315,14 @@ in
# hive declaring the vhost would answer for a service it does not have. # hive declaring the vhost would answer for a service it does not have.
services.hyperhive.gateway.localNames = [ cfg.domain ]; services.hyperhive.gateway.localNames = [ cfg.domain ];
# The secret oneshots as well as grafana itself: each runs before it and
# fails in ways grafana then reports only as a login that does not work.
services.hyperhive.swarm.otel.journaldUnits = [
"grafana"
"swarm-grafana-oidc-secret"
"swarm-grafana-secret-key"
];
services.hyperhive.swarm.controller.links = [ services.hyperhive.swarm.controller.links = [
{ {
label = "Grafana"; label = "Grafana";

View file

@ -408,6 +408,13 @@ in
}; };
config = lib.mkIf cfg.enable { config = lib.mkIf cfg.enable {
# The responder as well as the server: a denial reaches the client as a
# timeout, so the server's own log is the only place it is an error.
services.hyperhive.swarm.otel.journaldUnits = [
"nats"
"swarm-nats-auth"
];
assertions = [ assertions = [
{ {
# Fail at EVAL, not at boot: a queue that comes up unable to # Fail at EVAL, not at boot: a queue that comes up unable to

View file

@ -336,6 +336,32 @@ in
''; '';
}; };
journaldUnits = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
example = [ "nginx" ];
description = ''
systemd units whose journal this collector ships to the swarm's
log store. Nothing outside this list is collected.
Every module that defines a unit worth reading swarm-wide adds
its own names here, rather than one list naming them all: a
service that is not running contributes nothing, and a service
added later arrives with its units already declared. A central
list would be a second place that has to know which services
exist, and it would go stale in the direction that hides a
service's logs rather than the one that shows too many.
Names are matched as journald `_SYSTEMD_UNIT` values, so an
in-container unit is named exactly as it is inside its container
— `authelia`, not `container@swarm-authelia`.
⚠️ A name that matches nothing is not an error anywhere. The unit
simply never appears in the store, which looks the same as a
service that had nothing to say.
'';
};
clientId = lib.mkOption { clientId = lib.mkOption {
type = lib.types.str; type = lib.types.str;
readOnly = true; readOnly = true;
@ -362,6 +388,12 @@ in
# vhost would answer for a service it does not have. # vhost would answer for a service it does not have.
services.hyperhive.gateway.localNames = [ cfg.domain ]; services.hyperhive.gateway.localNames = [ cfg.domain ];
# This collector reads its own journal, so a pipeline that stops
# delivering says so in the store it stopped delivering to. That is
# less circular than it sounds: the failure that matters here is a
# single hive's receiver refusing pushes, not the process dying.
services.hyperhive.swarm.otel.journaldUnits = [ "opentelemetry-collector" ];
# OTLP/HTTP, not a browsable UI, but the same reverse-proxy shape as # OTLP/HTTP, not a browsable UI, but the same reverse-proxy shape as
# every sibling swarm service: TLS terminates here, then plain http to # every sibling swarm service: TLS terminates here, then plain http to
# the co-located container over loopback (shared netns, like the store # the co-located container over loopback (shared netns, like the store
@ -518,6 +550,24 @@ in
metrics store. metrics store.
''; '';
} }
{
# An empty allowlist is not an empty collection: the receiver
# renders no unit filter at all and reads the host's entire
# journal, an operator's desktop session included. Fail-open, and
# silent — so the one configuration that must not be reachable is
# "logs are being shipped and nothing said which".
assertion = !collectLogs || cfg.journaldUnits != [ ];
message = ''
The swarm collector has a log destination but
services.hyperhive.swarm.otel.journaldUnits is empty. An empty
list is not "collect nothing" — it is no filter at all, and this
collector would ship every unit on the host, including any
operator user session, to a store the whole swarm can read.
Name the units worth collecting, or turn off log collection by
leaving both log destinations unset.
'';
}
{ {
# Without a roster there are no receivers at all, so this # Without a roster there are no receivers at all, so this
# collector would listen on nothing while looking configured. # collector would listen on nothing while looking configured.
@ -825,23 +875,32 @@ in
} }
) cfg.publishedScrapeTargets; ) cfg.publishedScrapeTargets;
} }
# The whole journal directory, deliberately unfiltered. # The host's whole journal directory, read through a unit
# allowlist.
# #
# A filter restricted to the swarm's own containers was the # The directory is the host's rather than the swarm
# obvious shape and is the wrong one: the host units are where # containers', because the host units are where the incidents
# the incidents live — the gateway's nginx, the core daemon, # live — the gateway's nginx, the core daemon, dnsmasq — and
# dnsmasq — and none of them is a swarm container. Filtering to # none of them is a swarm container. A source restricted to
# `swarm-*` would exclude the single most-needed source. # `swarm-*` would exclude the most-needed one.
# #
# Nor would a list of container names stay true: it is a second # But that directory also holds every other unit on the host,
# place that has to know which services exist, and it goes stale # including an operator's `user@.service` session on a hive
# silently the next time one is added. Attribution does not need # that runs its services on a workstation. The allowlist is
# it — journald's `_HOSTNAME` / `_SYSTEMD_UNIT` / `_MACHINE_ID` # what separates "this host's services" from "this host", and
# are written by journald rather than by the logging process, so # only a per-unit one can: the receiver has no system-only
# a reader can tell the origins apart without this collector # switch, and its `matches` field is an allowlist too, so
# deciding for them. # "everything except user sessions" is not expressible.
#
# Attribution is journald's either way — `_HOSTNAME`,
# `_SYSTEMD_UNIT` and `_MACHINE_ID` are written by journald
# rather than by the logging process, so a reader can still
# tell the origins apart.
// lib.optionalAttrs collectLogs { // lib.optionalAttrs collectLogs {
journald.directory = hostJournalDir; journald = {
directory = hostJournalDir;
units = cfg.journaldUnits;
};
}; };
exporters = exporters =

View file

@ -147,6 +147,10 @@ in
# `swarm-victoriametrics.nix`). # `swarm-victoriametrics.nix`).
services.hyperhive.gateway.localNames = [ cfg.domain ]; services.hyperhive.gateway.localNames = [ cfg.domain ];
# Declared here rather than in the collector, so this store's logs are
# collected because it runs, not because a list elsewhere remembered it.
services.hyperhive.swarm.otel.journaldUnits = [ "victorialogs" ];
services.hyperhive.swarm.controller.links = [ services.hyperhive.swarm.controller.links = [
{ {
label = "Logs"; label = "Logs";

View file

@ -105,6 +105,10 @@ in
# have. # have.
services.hyperhive.gateway.localNames = [ cfg.domain ]; services.hyperhive.gateway.localNames = [ cfg.domain ];
# Declared here rather than in the collector, so this store's logs are
# collected because it runs, not because a list elsewhere remembered it.
services.hyperhive.swarm.otel.journaldUnits = [ "victoriametrics" ];
services.hyperhive.swarm.controller.links = [ services.hyperhive.swarm.controller.links = [
{ {
label = "Metrics"; label = "Metrics";