2977dc91d1
Adds services.hyperhive.otel.upstreamSignals, a list of "metrics" and/or "logs" naming the signals the swarm collector sends to otel.endpoint. The swarm's own stores always receive everything. The upstream exporter is only defined when at least one signal uses it, since otelcol refuses an exporter no pipeline references. On deploy: nothing changes until the option is set; the default is [ "metrics" "logs" ], today's behaviour. To stop the 404 flood from an upstream that has no /v1/logs, set services.hyperhive.otel.upstreamSignals = [ "metrics" ].
377 lines
17 KiB
Nix
377 lines
17 KiB
Nix
# `checks.module-eval-swarm-otel-core` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
shared,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
carriesJournaldSeverity
|
|
runGroup
|
|
otelSettings
|
|
;
|
|
|
|
# A swarm collector on a host that runs NEITHER store — the fully-spread
|
|
# shape from docs/swarm/services.md, and the one the old per-host gates made
|
|
# inexpressible. It is the whole point of the cases below that this hive is
|
|
# not a degenerate configuration but a supported one.
|
|
mkOtelNoStores =
|
|
upstreamSignals:
|
|
hive {
|
|
deploy.swarm-otel.enable = true;
|
|
deploy.authelia.enable = true;
|
|
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
|
|
deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem";
|
|
deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem";
|
|
deploy.victoriametrics.enable = false;
|
|
deploy.victorialogs.enable = false;
|
|
# An upstream makes `logExporterNames` longer than the store alone, which
|
|
# is what the self-journal case below needs to tell the two apart.
|
|
otel.endpoint = "https://upstream.example/otel";
|
|
otel.upstreamSignals = upstreamSignals;
|
|
};
|
|
otelNoStores = mkOtelNoStores [
|
|
"metrics"
|
|
"logs"
|
|
];
|
|
|
|
# The same collector with the upstream limited to one signal, and with none.
|
|
otelUpstreamMetricsOnly = mkOtelNoStores [ "metrics" ];
|
|
otelUpstreamNone = mkOtelNoStores [ ];
|
|
|
|
# The collector beside authelia, reading its own OIDC secret out of the
|
|
# store like every other collector — the cert pair here is not scenery, it
|
|
# is the arm that would catch the deleted co-located copy unit coming back.
|
|
otelBaoWithAuthelia = hive {
|
|
deploy.swarm-otel.enable = true;
|
|
deploy.authelia.enable = true;
|
|
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
|
|
deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem";
|
|
deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem";
|
|
};
|
|
|
|
# The same collector with the IdP on ANOTHER host and a store leaf placed by
|
|
# hand. Identical to the fixture above in everything the delivery path
|
|
# reads, which is the point.
|
|
otelBaoRemoteAuthelia = shared.otelReaderNamedInFull;
|
|
|
|
cases = [
|
|
{
|
|
# The host journal is read whole, so the filter is the only thing
|
|
# keeping an operator's desktop session out of the store. A processor a
|
|
# pipeline names but the config does not define is a collector that
|
|
# refuses to start, hence both halves.
|
|
name = "the host journal ships every unit, through the user-session filter";
|
|
ok =
|
|
let
|
|
s = otelSettings otelNoStores;
|
|
journalPipelines = lib.filter (p: builtins.elem "journald" p.receivers) (
|
|
lib.attrValues s.service.pipelines
|
|
);
|
|
in
|
|
!(s.receivers.journald ? units)
|
|
&& journalPipelines != [ ]
|
|
&& lib.all (p: builtins.elem "filter/exclude-user-sessions" p.processors) journalPipelines
|
|
&& s.processors."filter/exclude-user-sessions".logs.log_record or [ ] != [ ];
|
|
}
|
|
{
|
|
# 🩸 The arm that guards the ruling this slice landed under, the
|
|
# collector's half of ./swarm-grafana.nix's own. There is ONE delivery
|
|
# route: the store reader, on every host that runs the collector and
|
|
# holds a store identity. The negative names the deleted unit rather
|
|
# than a generic absence, because the way this regresses is someone
|
|
# re-adding the co-located copy as an optimisation.
|
|
name = "the collector's OIDC secret has exactly one delivery unit, the store reader, in both topologies";
|
|
ok =
|
|
let
|
|
local = otelBaoWithAuthelia.systemd.services;
|
|
remote = otelBaoRemoteAuthelia.systemd.services;
|
|
in
|
|
local ? swarm-bao-otel-oidc
|
|
&& remote ? swarm-bao-otel-oidc
|
|
&& !(local ? swarm-otel-oidc-secret)
|
|
&& !(remote ? swarm-otel-oidc-secret);
|
|
}
|
|
{
|
|
# Same 403-not-a-miss reason as grafana's arm above: the reader's grant
|
|
# covers the `services` prefix, so a path outside it is refused rather
|
|
# than empty, however correct it reads.
|
|
name = "the collector's OIDC secret is read from the prefix the publisher writes";
|
|
ok =
|
|
let
|
|
s = otelBaoRemoteAuthelia.systemd.services.swarm-bao-otel-oidc.script;
|
|
in
|
|
lib.hasInfix "secret/swarm/services/swarm-collector/oidc/client" s
|
|
&& !(lib.hasInfix "secret/swarm/hives/" s);
|
|
}
|
|
{
|
|
# A copy on persistent disk outlives the secret's rotation and lands in
|
|
# backups of the container's root.
|
|
name = "the collector's OIDC secret is written to the host's /run only, and the old copy removed";
|
|
ok =
|
|
let
|
|
m = otelBaoRemoteAuthelia;
|
|
script = m.systemd.services.swarm-bao-otel-oidc.script;
|
|
mount = m.containers.swarm-otel.bindMounts."/run/swarm-otel-oidc";
|
|
rules = m.systemd.tmpfiles.rules;
|
|
in
|
|
lib.hasInfix "atomic_write_secret 0400 root:root /run/swarm-otel-oidc/swarm-collector.secret" script
|
|
&& !(lib.hasInfix "/var/lib/nixos-containers" script)
|
|
&& !(lib.hasInfix "/var/lib/swarm-otel-oidc" script)
|
|
&& mount.hostPath == "/run/swarm-otel-oidc"
|
|
&& mount.isReadOnly
|
|
&&
|
|
m.services.hyperhive.deploy.swarm-otel.clientSecretFile
|
|
== "/run/swarm-otel-oidc/swarm-collector.secret"
|
|
&& lib.elem "d /run/swarm-otel-oidc 0700 root root -" rules
|
|
&& lib.elem "f /run/swarm-otel-oidc/swarm-collector.secret 0400 root root -" rules
|
|
&& lib.elem "r /var/lib/nixos-containers/swarm-otel/var/lib/swarm-otel-oidc/swarm-collector.secret" rules
|
|
&&
|
|
lib.any (c: c == "oidc-client-secret:/run/swarm-otel-oidc/swarm-collector.secret")
|
|
m.containers.swarm-otel.config.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential;
|
|
}
|
|
{
|
|
# These exporters must not be gated on the stores' PER-HOST enables: a
|
|
# collector that did not share a host with them would render none at all
|
|
# and drop everything it received, from every hive — silently, because
|
|
# an absent exporter is not an error.
|
|
name = "a collector that hosts neither store still exports to both";
|
|
ok =
|
|
let
|
|
e = (otelSettings otelNoStores).exporters;
|
|
in
|
|
(e ? "otlphttp/victoriametrics") && (e ? "otlphttp/victorialogs");
|
|
}
|
|
{
|
|
# A swarm has one of each store, so the address is a swarm-level name.
|
|
# A loopback literal here is the co-location assumption written back in,
|
|
# and it renders, deploys and reports healthy while reaching nothing.
|
|
name = "the store exporters address the stores by name, never by loopback";
|
|
ok =
|
|
let
|
|
e = (otelSettings otelNoStores).exporters;
|
|
m = e."otlphttp/victoriametrics".metrics_endpoint;
|
|
l = e."otlphttp/victorialogs".logs_endpoint;
|
|
in
|
|
!(lib.hasInfix "127.0.0.1" m)
|
|
&& !(lib.hasInfix "127.0.0.1" l)
|
|
&& lib.hasInfix "metrics.t.local" m
|
|
&& lib.hasInfix "logs.t.local" l;
|
|
}
|
|
{
|
|
# `swarm.otel.audience` is built in ../host-modules/swarm-otel-service.nix,
|
|
# the push URLs the exporters use in ../host-modules/swarm-otel.nix. A URL
|
|
# the registration lacks is refused at the token endpoint with
|
|
# `invalid_target`, so every push fails while the collector reports
|
|
# healthy.
|
|
name = "the collector's registered audiences include both store push URLs it requests";
|
|
ok =
|
|
let
|
|
s = otelSettings otelBaoWithAuthelia;
|
|
registered = otelBaoWithAuthelia.services.hyperhive.swarm.otel.audience;
|
|
withoutQuery = url: lib.head (lib.splitString "?" url);
|
|
pushUrls = [
|
|
s.exporters."otlphttp/victoriametrics".metrics_endpoint
|
|
(withoutQuery s.exporters."otlphttp/victorialogs".logs_endpoint)
|
|
];
|
|
requested = map (n: s.extensions."oauth2client/${n}".endpoint_params.audience) [
|
|
"victoriametrics"
|
|
"victorialogs"
|
|
];
|
|
in
|
|
lib.all (u: builtins.elem u registered) pushUrls
|
|
&& lib.sort lib.lessThan requested == lib.sort lib.lessThan pushUrls;
|
|
}
|
|
{
|
|
# `_HOSTNAME` cannot separate machines on its own: a hostname is a
|
|
# config value two of them can share, and then every stream for a unit
|
|
# name merges into one.
|
|
name = "the log stream is keyed by machine, not only by a hostname every container shares";
|
|
ok =
|
|
let
|
|
l = (otelSettings otelNoStores).exporters."otlphttp/victorialogs".logs_endpoint;
|
|
field = f: lib.hasInfix ("_stream_fields=" + f) l || lib.hasInfix ("," + f) l;
|
|
in
|
|
field "_MACHINE_ID" && field "_SYSTEMD_UNIT" && !(field "_NOSUCHFIELD");
|
|
}
|
|
{
|
|
# Defining an exporter and REFERENCING it are two separate lists, and
|
|
# the second is where the original gate also lived. An exporter no
|
|
# pipeline names is as silent as one that does not exist — this case
|
|
# exists because a mutation that restored only the reference-side gate
|
|
# left every other case here green.
|
|
name = "every pipeline that has a store exporter defined actually sends to it";
|
|
ok =
|
|
let
|
|
s = otelSettings otelNoStores;
|
|
used = lib.unique (lib.concatMap (p: p.exporters) (lib.attrValues s.service.pipelines));
|
|
in
|
|
builtins.elem "otlphttp/victoriametrics" used && builtins.elem "otlphttp/victorialogs" used;
|
|
}
|
|
{
|
|
# An authenticator an exporter names but `service.extensions` omits is
|
|
# INERT — the collector starts clean and pushes unauthenticated until
|
|
# something at the far end refuses it. Checked as a set relation rather
|
|
# than by naming the two, so it keeps holding for exporters not written
|
|
# yet.
|
|
name = "every exporter authenticator is listed in service.extensions";
|
|
ok =
|
|
let
|
|
s = otelSettings otelNoStores;
|
|
named = lib.filter (v: v != null) (
|
|
lib.mapAttrsToList (_: e: e.auth.authenticator or null) s.exporters
|
|
);
|
|
in
|
|
named != [ ] && lib.all (a: builtins.elem a s.service.extensions) named;
|
|
}
|
|
{
|
|
# The host-journal sibling of ./agent-otel.nix's wiring case, which
|
|
# carries the full reasoning. Same question, different receiver: this
|
|
# one reads the HOST's journal rather than a container's, and the two
|
|
# are unrelated config — a fixed stanza there, a parameterised block
|
|
# inside `containers.swarm-otel` here — so one losing its parser while
|
|
# the other keeps one is a real and silent state.
|
|
#
|
|
# Contents are not this case's business. The table both receivers import
|
|
# is asserted once, in ./journald-severity.nix.
|
|
name = "the swarm collector's journald receiver carries the shared PRIORITY mapping";
|
|
ok = carriesJournaldSeverity (otelSettings otelNoStores).receivers.journald;
|
|
}
|
|
{
|
|
# Sibling of ./hive-otel.nix's identical case for the host-tier
|
|
# collector: this one's `oidc/*` extensions call out to authelia at
|
|
# startup, so it fails the same way when that restart races its own.
|
|
# nixpkgs ships `Restart = "always"` with no `RestartSec`, so without
|
|
# this the unit burns its five default attempts inside two seconds and
|
|
# lands in `start-limit-hit`, where it stops retrying. The third clause
|
|
# is the one that has to hold — `StartLimit*` are `[Unit]` settings
|
|
# that systemd ignores under `[Service]`, so a bound written into
|
|
# `serviceConfig` renders, deploys and does nothing.
|
|
name = "the swarm collector backs off a failed bind from [Unit], not [Service]";
|
|
ok =
|
|
let
|
|
u = otelNoStores.containers.swarm-otel.config.systemd.services.opentelemetry-collector;
|
|
in
|
|
u.serviceConfig.RestartSec or 0 > 0
|
|
&& toString u.unitConfig.StartLimitBurst == "12"
|
|
&& !(u.serviceConfig ? StartLimitBurst)
|
|
# The window has to outlast every attempt, or the burst is unreachable.
|
|
&& u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst;
|
|
}
|
|
{
|
|
# 🩸 The container's own journal reaches the stores from the collector
|
|
# itself. Mounting the host's journal over the guest's /var/log/journal
|
|
# (or `--link-journal=host`) leaves `journald/self` reading nothing while
|
|
# the collector looks healthy. Presence controls: the host receiver and
|
|
# its mount are asserted too, so a misspelt path cannot pass.
|
|
name = "the swarm collector ships its own journal, and the host's from a mount that leaves it alone";
|
|
ok =
|
|
let
|
|
c = otelNoStores.containers.swarm-otel;
|
|
s = otelSettings otelNoStores;
|
|
self = lib.filter (p: builtins.elem "journald/self" p.receivers) (
|
|
lib.attrValues s.service.pipelines
|
|
);
|
|
in
|
|
!(lib.elem "--link-journal=host" (c.extraFlags or [ ]))
|
|
&& s.receivers."journald/self".directory == "/var/log/journal"
|
|
&& s.receivers."journald/self".merge
|
|
&& s.receivers.journald.directory == "/run/host-journal"
|
|
&& c.bindMounts."/run/host-journal".hostPath == "/var/log/journal"
|
|
&& !(c.bindMounts ? "/var/log/journal")
|
|
&& lib.length self == 1
|
|
&& lib.all (p: p.exporters == [ "otlphttp/victorialogs" ]) self
|
|
&& lib.elem "resource/self-journal" (lib.head self).processors
|
|
&& builtins.elem "resource/self-journal" (lib.attrNames s.processors);
|
|
}
|
|
{
|
|
# 🩸 The collector logs its own export failures to the journal that
|
|
# `journald/self` reads. Shipping that to the upstream feeds each
|
|
# failure back into the failing exporter. Presence controls: the
|
|
# upstream exporter exists and the host-journal pipeline does send to
|
|
# it, so an exporter-name typo cannot make the negative pass.
|
|
name = "the collector's own journal never reaches the upstream, and drops the store exporter's own failures";
|
|
ok =
|
|
let
|
|
s = otelSettings otelNoStores;
|
|
pipelines = lib.attrValues s.service.pipelines;
|
|
self = lib.filter (p: builtins.elem "journald/self" p.receivers) pipelines;
|
|
host = lib.filter (p: builtins.elem "journald" p.receivers) pipelines;
|
|
f = s.processors."filter/drop-store-export-failures";
|
|
in
|
|
s.exporters ? otlphttp
|
|
&& lib.length host == 1
|
|
&& lib.all (p: builtins.elem "otlphttp" p.exporters) host
|
|
&& lib.length self == 1
|
|
&& !(lib.any (p: builtins.elem "otlphttp" p.exporters) self)
|
|
&& lib.elem "filter/drop-store-export-failures" (lib.head self).processors
|
|
&& lib.length f.logs.log_record == 1
|
|
&& lib.hasInfix "Exporting failed" (lib.head f.logs.log_record)
|
|
&& lib.hasInfix "otlphttp/victorialogs" (lib.head f.logs.log_record);
|
|
}
|
|
{
|
|
# Presence control: the default sends both signals to the upstream, so
|
|
# the negative arms below cannot pass on a misspelled exporter name.
|
|
name = "the upstream receives metrics and logs by default";
|
|
ok =
|
|
let
|
|
s = otelSettings otelNoStores;
|
|
ps = s.service.pipelines;
|
|
names = pre: lib.filter (lib.hasPrefix pre) (lib.attrNames ps);
|
|
in
|
|
s.exporters ? otlphttp
|
|
&& names "metrics/" != [ ]
|
|
&& names "logs/" != [ ]
|
|
&& lib.all (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "metrics/")
|
|
&& lib.all (n: n == "logs/self" || builtins.elem "otlphttp" ps.${n}.exporters) (names "logs/");
|
|
}
|
|
{
|
|
name = "upstreamSignals limits which pipelines export upstream, and the stores keep everything";
|
|
ok =
|
|
let
|
|
s = otelSettings otelUpstreamMetricsOnly;
|
|
ps = s.service.pipelines;
|
|
names = pre: lib.filter (lib.hasPrefix pre) (lib.attrNames ps);
|
|
in
|
|
s.exporters ? otlphttp
|
|
&& names "metrics/" != [ ]
|
|
&& names "logs/" != [ ]
|
|
&& lib.all (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "metrics/")
|
|
&& lib.all (n: builtins.elem "otlphttp/victoriametrics" ps.${n}.exporters) (names "metrics/")
|
|
&& !(lib.any (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "logs/"))
|
|
&& lib.all (n: builtins.elem "otlphttp/victorialogs" ps.${n}.exporters) (names "logs/");
|
|
}
|
|
{
|
|
# The collector rejects an exporter no pipeline uses, so with nothing
|
|
# going upstream the exporter must not be defined.
|
|
name = "an empty upstreamSignals defines no upstream exporter and keeps the stores";
|
|
ok =
|
|
let
|
|
s = otelSettings otelUpstreamNone;
|
|
ps = lib.attrValues s.service.pipelines;
|
|
in
|
|
!(s.exporters ? otlphttp)
|
|
&& !(lib.any (p: builtins.elem "otlphttp" p.exporters) ps)
|
|
&& s.exporters ? "otlphttp/victoriametrics"
|
|
&& s.exporters ? "otlphttp/victorialogs"
|
|
&& lib.any (p: builtins.elem "otlphttp/victorialogs" p.exporters) ps
|
|
&& lib.any (p: builtins.elem "otlphttp/victoriametrics" p.exporters) ps;
|
|
}
|
|
];
|
|
in
|
|
runGroup "swarm-otel-core" cases
|