swarm-otel: collect only the units the swarm's services declare
The journald receiver was configured with a directory and no filter, so the swarm's log store received every unit on the host that runs the collector. On a hive whose services live on a workstation that includes the operator's desktop session, in a store every swarm operator can read. The receiver has no system-only switch and its `matches` field is an allowlist too, so what the swarm collects has to be stated rather than excluded. Each service module names its own units: a service that is not running contributes nothing, and one added later arrives declared. An empty list is fail-open — the receiver renders no filter at all and reads everything — so it is asserted against.
This commit is contained in:
parent
8879225fa4
commit
4336436457
11 changed files with 145 additions and 14 deletions
|
|
@ -136,6 +136,14 @@ in
|
||||||
];
|
];
|
||||||
|
|
||||||
config = lib.mkIf cfg.enable {
|
config = lib.mkIf cfg.enable {
|
||||||
|
# The daemon that owns every container on this hive, and the helper it
|
||||||
|
# delegates its root operations to. An agent asking why a container did
|
||||||
|
# not come up is asking about one of these two.
|
||||||
|
services.hyperhive.swarm.otel.journaldUnits = [
|
||||||
|
"hive-c0re"
|
||||||
|
"hive-priv"
|
||||||
|
];
|
||||||
|
|
||||||
assertions = [
|
assertions = [
|
||||||
{
|
{
|
||||||
# The pinned claude reaches agents as a bare path, so this
|
# The pinned claude reaches agents as a bare path, so this
|
||||||
|
|
|
||||||
|
|
@ -420,6 +420,14 @@ in
|
||||||
};
|
};
|
||||||
|
|
||||||
config = lib.mkIf config.services.hyperhive.enable {
|
config = lib.mkIf config.services.hyperhive.enable {
|
||||||
|
# Same principle as the vhost below — this service's own surface lives
|
||||||
|
# with the service. The SSO source is named alongside forgejo because
|
||||||
|
# its failure mode is a login that silently falls back, not an error.
|
||||||
|
services.hyperhive.swarm.otel.journaldUnits = [
|
||||||
|
"forgejo"
|
||||||
|
"forgejo-sso-source"
|
||||||
|
];
|
||||||
|
|
||||||
# This service's own gateway surface: the vhost that fronts it and
|
# This service's own gateway surface: the vhost that fronts it and
|
||||||
# the name the hive resolver answers for. Declared here rather than
|
# the name the hive resolver answers for. Declared here rather than
|
||||||
# in the gateway so the forge's public face lives with the forge —
|
# in the gateway so the forge's public face lives with the forge —
|
||||||
|
|
|
||||||
|
|
@ -115,6 +115,18 @@ in
|
||||||
# file's `let` and are not option surface.
|
# file's `let` and are not option surface.
|
||||||
services.hyperhive.gateway.lib = vhostLib;
|
services.hyperhive.gateway.lib = vhostLib;
|
||||||
|
|
||||||
|
# Every request to every hyperhive service passes through here, so this
|
||||||
|
# is the one unit that can say a service was unreachable rather than
|
||||||
|
# merely quiet. Named even on hives that run no swarm collector: the
|
||||||
|
# option is inert unless one is collecting on this host.
|
||||||
|
#
|
||||||
|
# dnsmasq alongside it for the same reason one level down: a name that
|
||||||
|
# stops resolving presents as every client timing out at once.
|
||||||
|
services.hyperhive.swarm.otel.journaldUnits = [
|
||||||
|
"nginx"
|
||||||
|
"dnsmasq"
|
||||||
|
];
|
||||||
|
|
||||||
assertions = [
|
assertions = [
|
||||||
{
|
{
|
||||||
assertion = !(cfg.tls.acme.enable && cfg.tls.certDir != null);
|
assertion = !(cfg.tls.acme.enable && cfg.tls.certDir != null);
|
||||||
|
|
|
||||||
|
|
@ -983,6 +983,13 @@ in
|
||||||
# bridge at that wrong answer.
|
# bridge at that wrong answer.
|
||||||
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
||||||
|
|
||||||
|
# The bridge as well as authelia: it is the half that writes the identity
|
||||||
|
# store, and its refusals are returned to callers as a bare 401.
|
||||||
|
services.hyperhive.swarm.otel.journaldUnits = [
|
||||||
|
"authelia"
|
||||||
|
"swarm-authelia-bridge"
|
||||||
|
];
|
||||||
|
|
||||||
# Declared here rather than in the collector's module, per the option's
|
# Declared here rather than in the collector's module, per the option's
|
||||||
# own rule: an entry exists only where the service that named it runs.
|
# own rule: an entry exists only where the service that named it runs.
|
||||||
#
|
#
|
||||||
|
|
|
||||||
|
|
@ -138,6 +138,13 @@ in
|
||||||
};
|
};
|
||||||
|
|
||||||
config = lib.mkIf (hyperhiveCfg.enable && cfg.autoConfigure) {
|
config = lib.mkIf (hyperhiveCfg.enable && cfg.autoConfigure) {
|
||||||
|
# A CA that fails to issue is invisible until something makes a TLS call
|
||||||
|
# hours later, so these two oneshots are worth more than most services.
|
||||||
|
services.hyperhive.swarm.otel.journaldUnits = [
|
||||||
|
"swarm-ca"
|
||||||
|
"swarm-services-ca"
|
||||||
|
];
|
||||||
|
|
||||||
systemd.services.swarm-ca = {
|
systemd.services.swarm-ca = {
|
||||||
description = "Generate the swarm root CA when absent";
|
description = "Generate the swarm root CA when absent";
|
||||||
wantedBy = [ "multi-user.target" ];
|
wantedBy = [ "multi-user.target" ];
|
||||||
|
|
|
||||||
|
|
@ -443,6 +443,13 @@ in
|
||||||
};
|
};
|
||||||
|
|
||||||
config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) {
|
config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) {
|
||||||
|
# The daemon and the oneshot that mints its credential — the second one
|
||||||
|
# failing leaves the first running and unable to authenticate anywhere.
|
||||||
|
services.hyperhive.swarm.otel.journaldUnits = [
|
||||||
|
"swarm-controller"
|
||||||
|
"swarm-controller-credential"
|
||||||
|
];
|
||||||
|
|
||||||
users.users.swarm-controller = {
|
users.users.swarm-controller = {
|
||||||
isSystemUser = true;
|
isSystemUser = true;
|
||||||
group = "swarm-controller";
|
group = "swarm-controller";
|
||||||
|
|
|
||||||
|
|
@ -315,6 +315,14 @@ in
|
||||||
# hive declaring the vhost would answer for a service it does not have.
|
# hive declaring the vhost would answer for a service it does not have.
|
||||||
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
||||||
|
|
||||||
|
# The secret oneshots as well as grafana itself: each runs before it and
|
||||||
|
# fails in ways grafana then reports only as a login that does not work.
|
||||||
|
services.hyperhive.swarm.otel.journaldUnits = [
|
||||||
|
"grafana"
|
||||||
|
"swarm-grafana-oidc-secret"
|
||||||
|
"swarm-grafana-secret-key"
|
||||||
|
];
|
||||||
|
|
||||||
services.hyperhive.swarm.controller.links = [
|
services.hyperhive.swarm.controller.links = [
|
||||||
{
|
{
|
||||||
label = "Grafana";
|
label = "Grafana";
|
||||||
|
|
|
||||||
|
|
@ -408,6 +408,13 @@ in
|
||||||
};
|
};
|
||||||
|
|
||||||
config = lib.mkIf cfg.enable {
|
config = lib.mkIf cfg.enable {
|
||||||
|
# The responder as well as the server: a denial reaches the client as a
|
||||||
|
# timeout, so the server's own log is the only place it is an error.
|
||||||
|
services.hyperhive.swarm.otel.journaldUnits = [
|
||||||
|
"nats"
|
||||||
|
"swarm-nats-auth"
|
||||||
|
];
|
||||||
|
|
||||||
assertions = [
|
assertions = [
|
||||||
{
|
{
|
||||||
# Fail at EVAL, not at boot: a queue that comes up unable to
|
# Fail at EVAL, not at boot: a queue that comes up unable to
|
||||||
|
|
|
||||||
|
|
@ -336,6 +336,32 @@ in
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
|
journaldUnits = lib.mkOption {
|
||||||
|
type = lib.types.listOf lib.types.str;
|
||||||
|
default = [ ];
|
||||||
|
example = [ "nginx" ];
|
||||||
|
description = ''
|
||||||
|
systemd units whose journal this collector ships to the swarm's
|
||||||
|
log store. Nothing outside this list is collected.
|
||||||
|
|
||||||
|
Every module that defines a unit worth reading swarm-wide adds
|
||||||
|
its own names here, rather than one list naming them all: a
|
||||||
|
service that is not running contributes nothing, and a service
|
||||||
|
added later arrives with its units already declared. A central
|
||||||
|
list would be a second place that has to know which services
|
||||||
|
exist, and it would go stale in the direction that hides a
|
||||||
|
service's logs rather than the one that shows too many.
|
||||||
|
|
||||||
|
Names are matched as journald `_SYSTEMD_UNIT` values, so an
|
||||||
|
in-container unit is named exactly as it is inside its container
|
||||||
|
— `authelia`, not `container@swarm-authelia`.
|
||||||
|
|
||||||
|
⚠️ A name that matches nothing is not an error anywhere. The unit
|
||||||
|
simply never appears in the store, which looks the same as a
|
||||||
|
service that had nothing to say.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
clientId = lib.mkOption {
|
clientId = lib.mkOption {
|
||||||
type = lib.types.str;
|
type = lib.types.str;
|
||||||
readOnly = true;
|
readOnly = true;
|
||||||
|
|
@ -362,6 +388,12 @@ in
|
||||||
# vhost would answer for a service it does not have.
|
# vhost would answer for a service it does not have.
|
||||||
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
||||||
|
|
||||||
|
# This collector reads its own journal, so a pipeline that stops
|
||||||
|
# delivering says so in the store it stopped delivering to. That is
|
||||||
|
# less circular than it sounds: the failure that matters here is a
|
||||||
|
# single hive's receiver refusing pushes, not the process dying.
|
||||||
|
services.hyperhive.swarm.otel.journaldUnits = [ "opentelemetry-collector" ];
|
||||||
|
|
||||||
# OTLP/HTTP, not a browsable UI, but the same reverse-proxy shape as
|
# OTLP/HTTP, not a browsable UI, but the same reverse-proxy shape as
|
||||||
# every sibling swarm service: TLS terminates here, then plain http to
|
# every sibling swarm service: TLS terminates here, then plain http to
|
||||||
# the co-located container over loopback (shared netns, like the store
|
# the co-located container over loopback (shared netns, like the store
|
||||||
|
|
@ -518,6 +550,24 @@ in
|
||||||
metrics store.
|
metrics store.
|
||||||
'';
|
'';
|
||||||
}
|
}
|
||||||
|
{
|
||||||
|
# An empty allowlist is not an empty collection: the receiver
|
||||||
|
# renders no unit filter at all and reads the host's entire
|
||||||
|
# journal, an operator's desktop session included. Fail-open, and
|
||||||
|
# silent — so the one configuration that must not be reachable is
|
||||||
|
# "logs are being shipped and nothing said which".
|
||||||
|
assertion = !collectLogs || cfg.journaldUnits != [ ];
|
||||||
|
message = ''
|
||||||
|
The swarm collector has a log destination but
|
||||||
|
services.hyperhive.swarm.otel.journaldUnits is empty. An empty
|
||||||
|
list is not "collect nothing" — it is no filter at all, and this
|
||||||
|
collector would ship every unit on the host, including any
|
||||||
|
operator user session, to a store the whole swarm can read.
|
||||||
|
|
||||||
|
Name the units worth collecting, or turn off log collection by
|
||||||
|
leaving both log destinations unset.
|
||||||
|
'';
|
||||||
|
}
|
||||||
{
|
{
|
||||||
# Without a roster there are no receivers at all, so this
|
# Without a roster there are no receivers at all, so this
|
||||||
# collector would listen on nothing while looking configured.
|
# collector would listen on nothing while looking configured.
|
||||||
|
|
@ -825,23 +875,32 @@ in
|
||||||
}
|
}
|
||||||
) cfg.publishedScrapeTargets;
|
) cfg.publishedScrapeTargets;
|
||||||
}
|
}
|
||||||
# The whole journal directory, deliberately unfiltered.
|
# The host's whole journal directory, read through a unit
|
||||||
|
# allowlist.
|
||||||
#
|
#
|
||||||
# A filter restricted to the swarm's own containers was the
|
# The directory is the host's rather than the swarm
|
||||||
# obvious shape and is the wrong one: the host units are where
|
# containers', because the host units are where the incidents
|
||||||
# the incidents live — the gateway's nginx, the core daemon,
|
# live — the gateway's nginx, the core daemon, dnsmasq — and
|
||||||
# dnsmasq — and none of them is a swarm container. Filtering to
|
# none of them is a swarm container. A source restricted to
|
||||||
# `swarm-*` would exclude the single most-needed source.
|
# `swarm-*` would exclude the most-needed one.
|
||||||
#
|
#
|
||||||
# Nor would a list of container names stay true: it is a second
|
# But that directory also holds every other unit on the host,
|
||||||
# place that has to know which services exist, and it goes stale
|
# including an operator's `user@.service` session on a hive
|
||||||
# silently the next time one is added. Attribution does not need
|
# that runs its services on a workstation. The allowlist is
|
||||||
# it — journald's `_HOSTNAME` / `_SYSTEMD_UNIT` / `_MACHINE_ID`
|
# what separates "this host's services" from "this host", and
|
||||||
# are written by journald rather than by the logging process, so
|
# only a per-unit one can: the receiver has no system-only
|
||||||
# a reader can tell the origins apart without this collector
|
# switch, and its `matches` field is an allowlist too, so
|
||||||
# deciding for them.
|
# "everything except user sessions" is not expressible.
|
||||||
|
#
|
||||||
|
# Attribution is journald's either way — `_HOSTNAME`,
|
||||||
|
# `_SYSTEMD_UNIT` and `_MACHINE_ID` are written by journald
|
||||||
|
# rather than by the logging process, so a reader can still
|
||||||
|
# tell the origins apart.
|
||||||
// lib.optionalAttrs collectLogs {
|
// lib.optionalAttrs collectLogs {
|
||||||
journald.directory = hostJournalDir;
|
journald = {
|
||||||
|
directory = hostJournalDir;
|
||||||
|
units = cfg.journaldUnits;
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
exporters =
|
exporters =
|
||||||
|
|
|
||||||
|
|
@ -147,6 +147,10 @@ in
|
||||||
# `swarm-victoriametrics.nix`).
|
# `swarm-victoriametrics.nix`).
|
||||||
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
||||||
|
|
||||||
|
# Declared here rather than in the collector, so this store's logs are
|
||||||
|
# collected because it runs, not because a list elsewhere remembered it.
|
||||||
|
services.hyperhive.swarm.otel.journaldUnits = [ "victorialogs" ];
|
||||||
|
|
||||||
services.hyperhive.swarm.controller.links = [
|
services.hyperhive.swarm.controller.links = [
|
||||||
{
|
{
|
||||||
label = "Logs";
|
label = "Logs";
|
||||||
|
|
|
||||||
|
|
@ -105,6 +105,10 @@ in
|
||||||
# have.
|
# have.
|
||||||
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
||||||
|
|
||||||
|
# Declared here rather than in the collector, so this store's logs are
|
||||||
|
# collected because it runs, not because a list elsewhere remembered it.
|
||||||
|
services.hyperhive.swarm.otel.journaldUnits = [ "victoriametrics" ];
|
||||||
|
|
||||||
services.hyperhive.swarm.controller.links = [
|
services.hyperhive.swarm.controller.links = [
|
||||||
{
|
{
|
||||||
label = "Metrics";
|
label = "Metrics";
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue