8d70afd93e
The logs/self pipeline shipped the collector's own journal to the upstream exporter too. When the upstream failed, the collector logged the failure, journald/self read it, and logs/self sent it upstream again: ~8 lines/s. logs/self now exports only to otlphttp/victorialogs, and a filter processor drops the collector's failure lines that name that exporter, so the store exporter cannot rebuild the loop one hop later. Other collector lines, including upstream failures, still reach the store. On deploy, swarm-otel's store rows drop from ~8/s of error spam to its real output.
318 lines
15 KiB
Nix
318 lines
15 KiB
Nix
# `checks.module-eval-swarm-otel-core` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
shared,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
carriesJournaldSeverity
|
|
runGroup
|
|
otelSettings
|
|
;
|
|
|
|
# A swarm collector on a host that runs NEITHER store — the fully-spread
|
|
# shape from docs/swarm/services.md, and the one the old per-host gates made
|
|
# inexpressible. It is the whole point of the cases below that this hive is
|
|
# not a degenerate configuration but a supported one.
|
|
otelNoStores = hive {
|
|
deploy.swarm-otel.enable = true;
|
|
deploy.authelia.enable = true;
|
|
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
|
|
deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem";
|
|
deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem";
|
|
deploy.victoriametrics.enable = false;
|
|
deploy.victorialogs.enable = false;
|
|
# An upstream makes `logExporterNames` longer than the store alone, which
|
|
# is what the self-journal case below needs to tell the two apart.
|
|
otel.endpoint = "https://upstream.example/otel";
|
|
};
|
|
|
|
# The collector beside authelia, reading its own OIDC secret out of the
|
|
# store like every other collector — the cert pair here is not scenery, it
|
|
# is the arm that would catch the deleted co-located copy unit coming back.
|
|
otelBaoWithAuthelia = hive {
|
|
deploy.swarm-otel.enable = true;
|
|
deploy.authelia.enable = true;
|
|
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
|
|
deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem";
|
|
deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem";
|
|
};
|
|
|
|
# The same collector with the IdP on ANOTHER host and a store leaf placed by
|
|
# hand. Identical to the fixture above in everything the delivery path
|
|
# reads, which is the point.
|
|
otelBaoRemoteAuthelia = shared.otelReaderNamedInFull;
|
|
|
|
cases = [
|
|
{
|
|
# The host journal is read whole, so the filter is the only thing
|
|
# keeping an operator's desktop session out of the store. A processor a
|
|
# pipeline names but the config does not define is a collector that
|
|
# refuses to start, hence both halves.
|
|
name = "the host journal ships every unit, through the user-session filter";
|
|
ok =
|
|
let
|
|
s = otelSettings otelNoStores;
|
|
journalPipelines = lib.filter (p: builtins.elem "journald" p.receivers) (
|
|
lib.attrValues s.service.pipelines
|
|
);
|
|
in
|
|
!(s.receivers.journald ? units)
|
|
&& journalPipelines != [ ]
|
|
&& lib.all (p: builtins.elem "filter/exclude-user-sessions" p.processors) journalPipelines
|
|
&& s.processors."filter/exclude-user-sessions".logs.log_record or [ ] != [ ];
|
|
}
|
|
{
|
|
# 🩸 The arm that guards the ruling this slice landed under, the
|
|
# collector's half of ./swarm-grafana.nix's own. There is ONE delivery
|
|
# route: the store reader, on every host that runs the collector and
|
|
# holds a store identity. The negative names the deleted unit rather
|
|
# than a generic absence, because the way this regresses is someone
|
|
# re-adding the co-located copy as an optimisation.
|
|
name = "the collector's OIDC secret has exactly one delivery unit, the store reader, in both topologies";
|
|
ok =
|
|
let
|
|
local = otelBaoWithAuthelia.systemd.services;
|
|
remote = otelBaoRemoteAuthelia.systemd.services;
|
|
in
|
|
local ? swarm-bao-otel-oidc
|
|
&& remote ? swarm-bao-otel-oidc
|
|
&& !(local ? swarm-otel-oidc-secret)
|
|
&& !(remote ? swarm-otel-oidc-secret);
|
|
}
|
|
{
|
|
# Same 403-not-a-miss reason as grafana's arm above: the reader's grant
|
|
# covers the `services` prefix, so a path outside it is refused rather
|
|
# than empty, however correct it reads.
|
|
name = "the collector's OIDC secret is read from the prefix the publisher writes";
|
|
ok =
|
|
let
|
|
s = otelBaoRemoteAuthelia.systemd.services.swarm-bao-otel-oidc.script;
|
|
in
|
|
lib.hasInfix "secret/swarm/services/swarm-collector/oidc/client" s
|
|
&& !(lib.hasInfix "secret/swarm/hives/" s);
|
|
}
|
|
{
|
|
# A copy on persistent disk outlives the secret's rotation and lands in
|
|
# backups of the container's root.
|
|
name = "the collector's OIDC secret is written to the host's /run only, and the old copy removed";
|
|
ok =
|
|
let
|
|
m = otelBaoRemoteAuthelia;
|
|
script = m.systemd.services.swarm-bao-otel-oidc.script;
|
|
mount = m.containers.swarm-otel.bindMounts."/run/swarm-otel-oidc";
|
|
rules = m.systemd.tmpfiles.rules;
|
|
in
|
|
lib.hasInfix "atomic_write_secret 0400 root:root /run/swarm-otel-oidc/swarm-collector.secret" script
|
|
&& !(lib.hasInfix "/var/lib/nixos-containers" script)
|
|
&& !(lib.hasInfix "/var/lib/swarm-otel-oidc" script)
|
|
&& mount.hostPath == "/run/swarm-otel-oidc"
|
|
&& mount.isReadOnly
|
|
&&
|
|
m.services.hyperhive.deploy.swarm-otel.clientSecretFile
|
|
== "/run/swarm-otel-oidc/swarm-collector.secret"
|
|
&& lib.elem "d /run/swarm-otel-oidc 0700 root root -" rules
|
|
&& lib.elem "f /run/swarm-otel-oidc/swarm-collector.secret 0400 root root -" rules
|
|
&& lib.elem "r /var/lib/nixos-containers/swarm-otel/var/lib/swarm-otel-oidc/swarm-collector.secret" rules
|
|
&&
|
|
lib.any (c: c == "oidc-client-secret:/run/swarm-otel-oidc/swarm-collector.secret")
|
|
m.containers.swarm-otel.config.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential;
|
|
}
|
|
{
|
|
# These exporters must not be gated on the stores' PER-HOST enables: a
|
|
# collector that did not share a host with them would render none at all
|
|
# and drop everything it received, from every hive — silently, because
|
|
# an absent exporter is not an error.
|
|
name = "a collector that hosts neither store still exports to both";
|
|
ok =
|
|
let
|
|
e = (otelSettings otelNoStores).exporters;
|
|
in
|
|
(e ? "otlphttp/victoriametrics") && (e ? "otlphttp/victorialogs");
|
|
}
|
|
{
|
|
# A swarm has one of each store, so the address is a swarm-level name.
|
|
# A loopback literal here is the co-location assumption written back in,
|
|
# and it renders, deploys and reports healthy while reaching nothing.
|
|
name = "the store exporters address the stores by name, never by loopback";
|
|
ok =
|
|
let
|
|
e = (otelSettings otelNoStores).exporters;
|
|
m = e."otlphttp/victoriametrics".metrics_endpoint;
|
|
l = e."otlphttp/victorialogs".logs_endpoint;
|
|
in
|
|
!(lib.hasInfix "127.0.0.1" m)
|
|
&& !(lib.hasInfix "127.0.0.1" l)
|
|
&& lib.hasInfix "metrics.t.local" m
|
|
&& lib.hasInfix "logs.t.local" l;
|
|
}
|
|
{
|
|
# `swarm.otel.audience` is built in ../host-modules/swarm-otel-service.nix,
|
|
# the push URLs the exporters use in ../host-modules/swarm-otel.nix. A URL
|
|
# the registration lacks is refused at the token endpoint with
|
|
# `invalid_target`, so every push fails while the collector reports
|
|
# healthy.
|
|
name = "the collector's registered audiences include both store push URLs it requests";
|
|
ok =
|
|
let
|
|
s = otelSettings otelBaoWithAuthelia;
|
|
registered = otelBaoWithAuthelia.services.hyperhive.swarm.otel.audience;
|
|
withoutQuery = url: lib.head (lib.splitString "?" url);
|
|
pushUrls = [
|
|
s.exporters."otlphttp/victoriametrics".metrics_endpoint
|
|
(withoutQuery s.exporters."otlphttp/victorialogs".logs_endpoint)
|
|
];
|
|
requested = map (n: s.extensions."oauth2client/${n}".endpoint_params.audience) [
|
|
"victoriametrics"
|
|
"victorialogs"
|
|
];
|
|
in
|
|
lib.all (u: builtins.elem u registered) pushUrls
|
|
&& lib.sort lib.lessThan requested == lib.sort lib.lessThan pushUrls;
|
|
}
|
|
{
|
|
# `_HOSTNAME` cannot separate machines on its own: a hostname is a
|
|
# config value two of them can share, and then every stream for a unit
|
|
# name merges into one.
|
|
name = "the log stream is keyed by machine, not only by a hostname every container shares";
|
|
ok =
|
|
let
|
|
l = (otelSettings otelNoStores).exporters."otlphttp/victorialogs".logs_endpoint;
|
|
field = f: lib.hasInfix ("_stream_fields=" + f) l || lib.hasInfix ("," + f) l;
|
|
in
|
|
field "_MACHINE_ID" && field "_SYSTEMD_UNIT" && !(field "_NOSUCHFIELD");
|
|
}
|
|
{
|
|
# Defining an exporter and REFERENCING it are two separate lists, and
|
|
# the second is where the original gate also lived. An exporter no
|
|
# pipeline names is as silent as one that does not exist — this case
|
|
# exists because a mutation that restored only the reference-side gate
|
|
# left every other case here green.
|
|
name = "every pipeline that has a store exporter defined actually sends to it";
|
|
ok =
|
|
let
|
|
s = otelSettings otelNoStores;
|
|
used = lib.unique (lib.concatMap (p: p.exporters) (lib.attrValues s.service.pipelines));
|
|
in
|
|
builtins.elem "otlphttp/victoriametrics" used && builtins.elem "otlphttp/victorialogs" used;
|
|
}
|
|
{
|
|
# An authenticator an exporter names but `service.extensions` omits is
|
|
# INERT — the collector starts clean and pushes unauthenticated until
|
|
# something at the far end refuses it. Checked as a set relation rather
|
|
# than by naming the two, so it keeps holding for exporters not written
|
|
# yet.
|
|
name = "every exporter authenticator is listed in service.extensions";
|
|
ok =
|
|
let
|
|
s = otelSettings otelNoStores;
|
|
named = lib.filter (v: v != null) (
|
|
lib.mapAttrsToList (_: e: e.auth.authenticator or null) s.exporters
|
|
);
|
|
in
|
|
named != [ ] && lib.all (a: builtins.elem a s.service.extensions) named;
|
|
}
|
|
{
|
|
# The host-journal sibling of ./agent-otel.nix's wiring case, which
|
|
# carries the full reasoning. Same question, different receiver: this
|
|
# one reads the HOST's journal rather than a container's, and the two
|
|
# are unrelated config — a fixed stanza there, a parameterised block
|
|
# inside `containers.swarm-otel` here — so one losing its parser while
|
|
# the other keeps one is a real and silent state.
|
|
#
|
|
# Contents are not this case's business. The table both receivers import
|
|
# is asserted once, in ./journald-severity.nix.
|
|
name = "the swarm collector's journald receiver carries the shared PRIORITY mapping";
|
|
ok = carriesJournaldSeverity (otelSettings otelNoStores).receivers.journald;
|
|
}
|
|
{
|
|
# Sibling of ./hive-otel.nix's identical case for the host-tier
|
|
# collector: this one's `oidc/*` extensions call out to authelia at
|
|
# startup, so it fails the same way when that restart races its own.
|
|
# nixpkgs ships `Restart = "always"` with no `RestartSec`, so without
|
|
# this the unit burns its five default attempts inside two seconds and
|
|
# lands in `start-limit-hit`, where it stops retrying. The third clause
|
|
# is the one that has to hold — `StartLimit*` are `[Unit]` settings
|
|
# that systemd ignores under `[Service]`, so a bound written into
|
|
# `serviceConfig` renders, deploys and does nothing.
|
|
name = "the swarm collector backs off a failed bind from [Unit], not [Service]";
|
|
ok =
|
|
let
|
|
u = otelNoStores.containers.swarm-otel.config.systemd.services.opentelemetry-collector;
|
|
in
|
|
u.serviceConfig.RestartSec or 0 > 0
|
|
&& toString u.unitConfig.StartLimitBurst == "12"
|
|
&& !(u.serviceConfig ? StartLimitBurst)
|
|
# The window has to outlast every attempt, or the burst is unreachable.
|
|
&& u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst;
|
|
}
|
|
{
|
|
# 🩸 The container's own journal reaches the stores from the collector
|
|
# itself. Mounting the host's journal over the guest's /var/log/journal
|
|
# (or `--link-journal=host`) leaves `journald/self` reading nothing while
|
|
# the collector looks healthy. Presence controls: the host receiver and
|
|
# its mount are asserted too, so a misspelt path cannot pass.
|
|
name = "the swarm collector ships its own journal, and the host's from a mount that leaves it alone";
|
|
ok =
|
|
let
|
|
c = otelNoStores.containers.swarm-otel;
|
|
s = otelSettings otelNoStores;
|
|
self = lib.filter (p: builtins.elem "journald/self" p.receivers) (
|
|
lib.attrValues s.service.pipelines
|
|
);
|
|
in
|
|
!(lib.elem "--link-journal=host" (c.extraFlags or [ ]))
|
|
&& s.receivers."journald/self".directory == "/var/log/journal"
|
|
&& s.receivers."journald/self".merge
|
|
&& s.receivers.journald.directory == "/run/host-journal"
|
|
&& c.bindMounts."/run/host-journal".hostPath == "/var/log/journal"
|
|
&& !(c.bindMounts ? "/var/log/journal")
|
|
&& lib.length self == 1
|
|
&& lib.all (p: p.exporters == [ "otlphttp/victorialogs" ]) self
|
|
&& lib.elem "resource/self-journal" (lib.head self).processors
|
|
&& builtins.elem "resource/self-journal" (lib.attrNames s.processors);
|
|
}
|
|
{
|
|
# 🩸 The collector logs its own export failures to the journal that
|
|
# `journald/self` reads. Shipping that to the upstream feeds each
|
|
# failure back into the failing exporter. Presence controls: the
|
|
# upstream exporter exists and the host-journal pipeline does send to
|
|
# it, so an exporter-name typo cannot make the negative pass.
|
|
name = "the collector's own journal never reaches the upstream, and drops the store exporter's own failures";
|
|
ok =
|
|
let
|
|
s = otelSettings otelNoStores;
|
|
pipelines = lib.attrValues s.service.pipelines;
|
|
self = lib.filter (p: builtins.elem "journald/self" p.receivers) pipelines;
|
|
host = lib.filter (p: builtins.elem "journald" p.receivers) pipelines;
|
|
f = s.processors."filter/drop-store-export-failures";
|
|
in
|
|
s.exporters ? otlphttp
|
|
&& lib.length host == 1
|
|
&& lib.all (p: builtins.elem "otlphttp" p.exporters) host
|
|
&& lib.length self == 1
|
|
&& !(lib.any (p: builtins.elem "otlphttp" p.exporters) self)
|
|
&& lib.elem "filter/drop-store-export-failures" (lib.head self).processors
|
|
&& lib.length f.logs.log_record == 1
|
|
&& lib.hasInfix "Exporting failed" (lib.head f.logs.log_record)
|
|
&& lib.hasInfix "otlphttp/victorialogs" (lib.head f.logs.log_record);
|
|
}
|
|
];
|
|
in
|
|
runGroup "swarm-otel-core" cases
|