feat(swarm-otel): fan logs out like metrics, so the local store is optional
The logs half was gated on the swarm's log store being enabled, so turning that store off stopped collection entirely rather than leaving the upstream export. Logs now fan out exactly as metrics do: the store when it runs, the operator's upstream when one is configured, both when both. The receiver, the journal mount and the group grant follow whether there is anywhere to send logs, not whether the local store exists. Tying the mount to the store instead would render a receiver that can read nothing.
This commit is contained in:
parent
bfa25c6419
commit
f46ef39ef3
1 changed files with 31 additions and 14 deletions
|
|
@ -140,13 +140,29 @@ let
|
|||
# those subdirectories, so one reader covers the host and every container.
|
||||
hostJournalDir = "/var/log/journal";
|
||||
|
||||
# The operator-configured upstream, named once: the same exporter carries
|
||||
# every signal, so metrics and logs both reach it without a second
|
||||
# definition.
|
||||
upstreamExporterName = if otelCfg.protocol == "grpc" then "otlp" else "otlphttp";
|
||||
upstreamExporters = lib.optional (otelCfg.endpoint != "") upstreamExporterName;
|
||||
|
||||
# One list, read by every pipeline: the per-hive pipelines fan out to
|
||||
# exactly the same destinations as the single pipeline they replace.
|
||||
# Written once because "which exporters" is a property of this tier, not
|
||||
# of which hive a sample came from.
|
||||
exporterNames =
|
||||
lib.optional (otelCfg.endpoint != "") (if otelCfg.protocol == "grpc" then "otlp" else "otlphttp")
|
||||
++ lib.optional vmCfg.enable "otlphttp/victoriametrics";
|
||||
exporterNames = upstreamExporters ++ lib.optional vmCfg.enable "otlphttp/victoriametrics";
|
||||
|
||||
# The same fan-out for logs, and the local store is only ONE of its
|
||||
# destinations. A deployment that turns the swarm's log store off and keeps
|
||||
# an upstream endpoint still collects — the store is where logs may be kept,
|
||||
# not the reason to read the journal at all.
|
||||
logExporterNames = upstreamExporters ++ lib.optional vlCfg.enable "otlphttp/victorialogs";
|
||||
|
||||
# Collect when there is anywhere to send it, and only then. A pipeline with
|
||||
# an empty exporter list is not a quiet no-op — the collector rejects it —
|
||||
# and reading the journal to drop it on the floor would be worse than not
|
||||
# reading it.
|
||||
collectLogs = logExporterNames != [ ];
|
||||
in
|
||||
{
|
||||
options.services.hyperhive.swarm.otel = {
|
||||
|
|
@ -672,7 +688,7 @@ in
|
|||
# write to. Read-only is the whole security posture of this mount: the
|
||||
# collector has no business writing to a journal, and a collector that
|
||||
# cannot write cannot corrupt the record it is reporting on.
|
||||
// lib.optionalAttrs vlCfg.enable {
|
||||
// lib.optionalAttrs collectLogs {
|
||||
${hostJournalDir} = {
|
||||
hostPath = hostJournalDir;
|
||||
isReadOnly = true;
|
||||
|
|
@ -811,7 +827,7 @@ in
|
|||
# are written by journald rather than by the logging process, so
|
||||
# a reader can tell the origins apart without this collector
|
||||
# deciding for them.
|
||||
// lib.optionalAttrs vlCfg.enable {
|
||||
// lib.optionalAttrs collectLogs {
|
||||
journald.directory = hostJournalDir;
|
||||
};
|
||||
|
||||
|
|
@ -926,19 +942,20 @@ in
|
|||
exporters = exporterNames;
|
||||
};
|
||||
}
|
||||
# Logs go to the swarm's store and NOT to `exporterNames`, so
|
||||
# they do not follow the metrics upstream. That asymmetry is
|
||||
# deliberate: an operator who configured an upstream endpoint
|
||||
# agreed to ship metrics there, and quietly adding their logs to
|
||||
# the same hop is a disclosure decision, not a symmetry fix.
|
||||
# Logs fan out exactly as metrics do: the swarm's store when it
|
||||
# runs, the operator's upstream when one is configured, both
|
||||
# when both. The local store is a destination rather than the
|
||||
# reason to collect, so turning it off leaves a hive that still
|
||||
# ships its journal to whoever aggregates across swarms.
|
||||
#
|
||||
# Same stamp as the scraped swarm services, for the same reason:
|
||||
# a host's logs belong to the swarm, and there is no honest
|
||||
# `hive` value to put on them.
|
||||
// lib.optionalAttrs vlCfg.enable {
|
||||
// lib.optionalAttrs collectLogs {
|
||||
"logs/${swarmTierName}" = {
|
||||
receivers = [ "journald" ];
|
||||
processors = [ "resource/${swarmTierName}" ];
|
||||
exporters = [ "otlphttp/victorialogs" ];
|
||||
exporters = logExporterNames;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -1030,7 +1047,7 @@ in
|
|||
# a collector that refuses to start, and enabling the log store
|
||||
# without declaring a scrape target is a perfectly ordinary
|
||||
# config.
|
||||
// lib.optionalAttrs (cfg.scrapeTargets != { } || vlCfg.enable) {
|
||||
// lib.optionalAttrs (cfg.scrapeTargets != { } || collectLogs) {
|
||||
"resource/${swarmTierName}".attributes = [
|
||||
{
|
||||
# The metric LABEL, a different namespace from the
|
||||
|
|
@ -1072,7 +1089,7 @@ in
|
|||
# host. A dynamically allocated gid would mean the host's ownership
|
||||
# named a different group inside the container, and the failure
|
||||
# would be a receiver that starts cleanly and reads nothing.
|
||||
// lib.optionalAttrs vlCfg.enable {
|
||||
// lib.optionalAttrs collectLogs {
|
||||
SupplementaryGroups = [ "systemd-journal" ];
|
||||
};
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in a new issue