swarm-otel: collect only the units the swarm's services declare
The journald receiver was configured with a directory and no filter, so the swarm's log store received every unit on the host that runs the collector. On a hive whose services live on a workstation that includes the operator's desktop session, in a store every swarm operator can read. The receiver has no system-only switch and its `matches` field is an allowlist too, so what the swarm collects has to be stated rather than excluded. Each service module names its own units: a service that is not running contributes nothing, and one added later arrives declared. An empty list is fail-open — the receiver renders no filter at all and reads everything — so it is asserted against.
This commit is contained in:
parent
8879225fa4
commit
4336436457
11 changed files with 145 additions and 14 deletions
|
|
@ -136,6 +136,14 @@ in
|
|||
];
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
# The daemon that owns every container on this hive, and the helper it
|
||||
# delegates its root operations to. An agent asking why a container did
|
||||
# not come up is asking about one of these two.
|
||||
services.hyperhive.swarm.otel.journaldUnits = [
|
||||
"hive-c0re"
|
||||
"hive-priv"
|
||||
];
|
||||
|
||||
assertions = [
|
||||
{
|
||||
# The pinned claude reaches agents as a bare path, so this
|
||||
|
|
|
|||
|
|
@ -420,6 +420,14 @@ in
|
|||
};
|
||||
|
||||
config = lib.mkIf config.services.hyperhive.enable {
|
||||
# Same principle as the vhost below — this service's own surface lives
|
||||
# with the service. The SSO source is named alongside forgejo because
|
||||
# its failure mode is a login that silently falls back, not an error.
|
||||
services.hyperhive.swarm.otel.journaldUnits = [
|
||||
"forgejo"
|
||||
"forgejo-sso-source"
|
||||
];
|
||||
|
||||
# This service's own gateway surface: the vhost that fronts it and
|
||||
# the name the hive resolver answers for. Declared here rather than
|
||||
# in the gateway so the forge's public face lives with the forge —
|
||||
|
|
|
|||
|
|
@ -115,6 +115,18 @@ in
|
|||
# file's `let` and are not option surface.
|
||||
services.hyperhive.gateway.lib = vhostLib;
|
||||
|
||||
# Every request to every hyperhive service passes through here, so this
|
||||
# is the one unit that can say a service was unreachable rather than
|
||||
# merely quiet. Named even on hives that run no swarm collector: the
|
||||
# option is inert unless one is collecting on this host.
|
||||
#
|
||||
# dnsmasq alongside it for the same reason one level down: a name that
|
||||
# stops resolving presents as every client timing out at once.
|
||||
services.hyperhive.swarm.otel.journaldUnits = [
|
||||
"nginx"
|
||||
"dnsmasq"
|
||||
];
|
||||
|
||||
assertions = [
|
||||
{
|
||||
assertion = !(cfg.tls.acme.enable && cfg.tls.certDir != null);
|
||||
|
|
|
|||
|
|
@ -983,6 +983,13 @@ in
|
|||
# bridge at that wrong answer.
|
||||
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
||||
|
||||
# The bridge as well as authelia: it is the half that writes the identity
|
||||
# store, and its refusals are returned to callers as a bare 401.
|
||||
services.hyperhive.swarm.otel.journaldUnits = [
|
||||
"authelia"
|
||||
"swarm-authelia-bridge"
|
||||
];
|
||||
|
||||
# Declared here rather than in the collector's module, per the option's
|
||||
# own rule: an entry exists only where the service that named it runs.
|
||||
#
|
||||
|
|
|
|||
|
|
@ -138,6 +138,13 @@ in
|
|||
};
|
||||
|
||||
config = lib.mkIf (hyperhiveCfg.enable && cfg.autoConfigure) {
|
||||
# A CA that fails to issue is invisible until something makes a TLS call
|
||||
# hours later, so these two oneshots are worth more than most services.
|
||||
services.hyperhive.swarm.otel.journaldUnits = [
|
||||
"swarm-ca"
|
||||
"swarm-services-ca"
|
||||
];
|
||||
|
||||
systemd.services.swarm-ca = {
|
||||
description = "Generate the swarm root CA when absent";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
|
|
|
|||
|
|
@ -443,6 +443,13 @@ in
|
|||
};
|
||||
|
||||
config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) {
|
||||
# The daemon and the oneshot that mints its credential — the second one
|
||||
# failing leaves the first running and unable to authenticate anywhere.
|
||||
services.hyperhive.swarm.otel.journaldUnits = [
|
||||
"swarm-controller"
|
||||
"swarm-controller-credential"
|
||||
];
|
||||
|
||||
users.users.swarm-controller = {
|
||||
isSystemUser = true;
|
||||
group = "swarm-controller";
|
||||
|
|
|
|||
|
|
@ -315,6 +315,14 @@ in
|
|||
# hive declaring the vhost would answer for a service it does not have.
|
||||
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
||||
|
||||
# The secret oneshots as well as grafana itself: each runs before it and
|
||||
# fails in ways grafana then reports only as a login that does not work.
|
||||
services.hyperhive.swarm.otel.journaldUnits = [
|
||||
"grafana"
|
||||
"swarm-grafana-oidc-secret"
|
||||
"swarm-grafana-secret-key"
|
||||
];
|
||||
|
||||
services.hyperhive.swarm.controller.links = [
|
||||
{
|
||||
label = "Grafana";
|
||||
|
|
|
|||
|
|
@ -408,6 +408,13 @@ in
|
|||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
# The responder as well as the server: a denial reaches the client as a
|
||||
# timeout, so the server's own log is the only place it is an error.
|
||||
services.hyperhive.swarm.otel.journaldUnits = [
|
||||
"nats"
|
||||
"swarm-nats-auth"
|
||||
];
|
||||
|
||||
assertions = [
|
||||
{
|
||||
# Fail at EVAL, not at boot: a queue that comes up unable to
|
||||
|
|
|
|||
|
|
@ -336,6 +336,32 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
journaldUnits = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
example = [ "nginx" ];
|
||||
description = ''
|
||||
systemd units whose journal this collector ships to the swarm's
|
||||
log store. Nothing outside this list is collected.
|
||||
|
||||
Every module that defines a unit worth reading swarm-wide adds
|
||||
its own names here, rather than one list naming them all: a
|
||||
service that is not running contributes nothing, and a service
|
||||
added later arrives with its units already declared. A central
|
||||
list would be a second place that has to know which services
|
||||
exist, and it would go stale in the direction that hides a
|
||||
service's logs rather than the one that shows too many.
|
||||
|
||||
Names are matched as journald `_SYSTEMD_UNIT` values, so an
|
||||
in-container unit is named exactly as it is inside its container
|
||||
— `authelia`, not `container@swarm-authelia`.
|
||||
|
||||
⚠️ A name that matches nothing is not an error anywhere. The unit
|
||||
simply never appears in the store, which looks the same as a
|
||||
service that had nothing to say.
|
||||
'';
|
||||
};
|
||||
|
||||
clientId = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
readOnly = true;
|
||||
|
|
@ -362,6 +388,12 @@ in
|
|||
# vhost would answer for a service it does not have.
|
||||
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
||||
|
||||
# This collector reads its own journal, so a pipeline that stops
|
||||
# delivering says so in the store it stopped delivering to. That is
|
||||
# less circular than it sounds: the failure that matters here is a
|
||||
# single hive's receiver refusing pushes, not the process dying.
|
||||
services.hyperhive.swarm.otel.journaldUnits = [ "opentelemetry-collector" ];
|
||||
|
||||
# OTLP/HTTP, not a browsable UI, but the same reverse-proxy shape as
|
||||
# every sibling swarm service: TLS terminates here, then plain http to
|
||||
# the co-located container over loopback (shared netns, like the store
|
||||
|
|
@ -518,6 +550,24 @@ in
|
|||
metrics store.
|
||||
'';
|
||||
}
|
||||
{
|
||||
# An empty allowlist is not an empty collection: the receiver
|
||||
# renders no unit filter at all and reads the host's entire
|
||||
# journal, an operator's desktop session included. Fail-open, and
|
||||
# silent — so the one configuration that must not be reachable is
|
||||
# "logs are being shipped and nothing said which".
|
||||
assertion = !collectLogs || cfg.journaldUnits != [ ];
|
||||
message = ''
|
||||
The swarm collector has a log destination but
|
||||
services.hyperhive.swarm.otel.journaldUnits is empty. An empty
|
||||
list is not "collect nothing" — it is no filter at all, and this
|
||||
collector would ship every unit on the host, including any
|
||||
operator user session, to a store the whole swarm can read.
|
||||
|
||||
Name the units worth collecting, or turn off log collection by
|
||||
leaving both log destinations unset.
|
||||
'';
|
||||
}
|
||||
{
|
||||
# Without a roster there are no receivers at all, so this
|
||||
# collector would listen on nothing while looking configured.
|
||||
|
|
@ -825,23 +875,32 @@ in
|
|||
}
|
||||
) cfg.publishedScrapeTargets;
|
||||
}
|
||||
# The whole journal directory, deliberately unfiltered.
|
||||
# The host's whole journal directory, read through a unit
|
||||
# allowlist.
|
||||
#
|
||||
# A filter restricted to the swarm's own containers was the
|
||||
# obvious shape and is the wrong one: the host units are where
|
||||
# the incidents live — the gateway's nginx, the core daemon,
|
||||
# dnsmasq — and none of them is a swarm container. Filtering to
|
||||
# `swarm-*` would exclude the single most-needed source.
|
||||
# The directory is the host's rather than the swarm
|
||||
# containers', because the host units are where the incidents
|
||||
# live — the gateway's nginx, the core daemon, dnsmasq — and
|
||||
# none of them is a swarm container. A source restricted to
|
||||
# `swarm-*` would exclude the most-needed one.
|
||||
#
|
||||
# Nor would a list of container names stay true: it is a second
|
||||
# place that has to know which services exist, and it goes stale
|
||||
# silently the next time one is added. Attribution does not need
|
||||
# it — journald's `_HOSTNAME` / `_SYSTEMD_UNIT` / `_MACHINE_ID`
|
||||
# are written by journald rather than by the logging process, so
|
||||
# a reader can tell the origins apart without this collector
|
||||
# deciding for them.
|
||||
# But that directory also holds every other unit on the host,
|
||||
# including an operator's `user@.service` session on a hive
|
||||
# that runs its services on a workstation. The allowlist is
|
||||
# what separates "this host's services" from "this host", and
|
||||
# only a per-unit one can: the receiver has no system-only
|
||||
# switch, and its `matches` field is an allowlist too, so
|
||||
# "everything except user sessions" is not expressible.
|
||||
#
|
||||
# Attribution is journald's either way — `_HOSTNAME`,
|
||||
# `_SYSTEMD_UNIT` and `_MACHINE_ID` are written by journald
|
||||
# rather than by the logging process, so a reader can still
|
||||
# tell the origins apart.
|
||||
// lib.optionalAttrs collectLogs {
|
||||
journald.directory = hostJournalDir;
|
||||
journald = {
|
||||
directory = hostJournalDir;
|
||||
units = cfg.journaldUnits;
|
||||
};
|
||||
};
|
||||
|
||||
exporters =
|
||||
|
|
|
|||
|
|
@ -147,6 +147,10 @@ in
|
|||
# `swarm-victoriametrics.nix`).
|
||||
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
||||
|
||||
# Declared here rather than in the collector, so this store's logs are
|
||||
# collected because it runs, not because a list elsewhere remembered it.
|
||||
services.hyperhive.swarm.otel.journaldUnits = [ "victorialogs" ];
|
||||
|
||||
services.hyperhive.swarm.controller.links = [
|
||||
{
|
||||
label = "Logs";
|
||||
|
|
|
|||
|
|
@ -105,6 +105,10 @@ in
|
|||
# have.
|
||||
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
||||
|
||||
# Declared here rather than in the collector, so this store's logs are
|
||||
# collected because it runs, not because a list elsewhere remembered it.
|
||||
services.hyperhive.swarm.otel.journaldUnits = [ "victoriametrics" ];
|
||||
|
||||
services.hyperhive.swarm.controller.links = [
|
||||
{
|
||||
label = "Metrics";
|
||||
|
|
|
|||
Loading…
Reference in a new issue