440 lines
20 KiB
Nix
440 lines
20 KiB
Nix
# `checks.module-eval-swarm-otel-core` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
shared,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
carriesJournaldSeverity
|
|
runGroup
|
|
otelSettings
|
|
;
|
|
|
|
# A swarm collector on a host that runs NEITHER store — the fully-spread
|
|
# shape from docs/swarm/services.md, and the one the old per-host gates made
|
|
# inexpressible. It is the whole point of the cases below that this hive is
|
|
# not a degenerate configuration but a supported one.
|
|
mkOtelNoStores =
|
|
upstreamSignals:
|
|
hive {
|
|
deploy.swarm-otel.enable = true;
|
|
deploy.authelia.enable = true;
|
|
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
|
|
deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem";
|
|
deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem";
|
|
deploy.victoriametrics.enable = false;
|
|
deploy.victorialogs.enable = false;
|
|
# An upstream makes `logExporterNames` longer than the store alone, which
|
|
# is what the self-journal case below needs to tell the two apart.
|
|
deploy.swarm-otel.upstream.endpoint = "https://upstream.example/otel";
|
|
deploy.swarm-otel.upstream.signals = upstreamSignals;
|
|
};
|
|
otelNoStores = mkOtelNoStores [
|
|
"metrics"
|
|
"logs"
|
|
];
|
|
|
|
# A host config still on the `otel.*` upstream paths, every shipped one.
|
|
otelOldUpstreamPaths = hive {
|
|
deploy.swarm-otel.enable = true;
|
|
deploy.authelia.enable = true;
|
|
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
|
|
deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem";
|
|
deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem";
|
|
otel.endpoint = "https://old.example/otel";
|
|
otel.protocol = "http/json";
|
|
otel.headersCredential = "/run/secrets/old-otel-headers";
|
|
otel.collector.upstreamHeaderName = "X-Old-Token";
|
|
};
|
|
|
|
# The same collector with the upstream limited to one signal, and with none.
|
|
otelUpstreamMetricsOnly = mkOtelNoStores [ "metrics" ];
|
|
otelUpstreamNone = mkOtelNoStores [ ];
|
|
|
|
# The collector beside authelia, reading its own OIDC secret out of the
|
|
# store like every other collector — the cert pair here is not scenery, it
|
|
# is the arm that would catch the deleted co-located copy unit coming back.
|
|
otelBaoWithAuthelia = hive {
|
|
deploy.swarm-otel.enable = true;
|
|
deploy.authelia.enable = true;
|
|
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
|
|
deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem";
|
|
deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem";
|
|
};
|
|
|
|
# The same collector with the IdP on ANOTHER host and a store leaf placed by
|
|
# hand. Identical to the fixture above in everything the delivery path
|
|
# reads, which is the point.
|
|
otelBaoRemoteAuthelia = shared.otelReaderNamedInFull;
|
|
|
|
cases = [
|
|
{
|
|
# The host journal is read whole, so the filter is the only thing
|
|
# keeping an operator's desktop session out of the store. A processor a
|
|
# pipeline names but the config does not define is a collector that
|
|
# refuses to start, hence both halves.
|
|
name = "the host journal ships every unit, through the user-session filter";
|
|
ok =
|
|
let
|
|
s = otelSettings otelNoStores;
|
|
journalPipelines = lib.filter (p: builtins.elem "journald" p.receivers) (
|
|
lib.attrValues s.service.pipelines
|
|
);
|
|
in
|
|
!(s.receivers.journald ? units)
|
|
&& journalPipelines != [ ]
|
|
&& lib.all (p: builtins.elem "filter/exclude-user-sessions" p.processors) journalPipelines
|
|
&& s.processors."filter/exclude-user-sessions".logs.log_record or [ ] != [ ];
|
|
}
|
|
{
|
|
# 🩸 The arm that guards the ruling this slice landed under, the
|
|
# collector's half of ./swarm-grafana.nix's own. There is ONE delivery
|
|
# route: the store reader, on every host that runs the collector and
|
|
# holds a store identity. The negative names the deleted unit rather
|
|
# than a generic absence, because the way this regresses is someone
|
|
# re-adding the co-located copy as an optimisation.
|
|
name = "the collector's OIDC secret has exactly one delivery unit, the store reader, in both topologies";
|
|
ok =
|
|
let
|
|
local = otelBaoWithAuthelia.systemd.services;
|
|
remote = otelBaoRemoteAuthelia.systemd.services;
|
|
in
|
|
local ? swarm-bao-otel-oidc
|
|
&& remote ? swarm-bao-otel-oidc
|
|
&& !(local ? swarm-otel-oidc-secret)
|
|
&& !(remote ? swarm-otel-oidc-secret);
|
|
}
|
|
{
|
|
# Same 403-not-a-miss reason as grafana's arm above: the reader's grant
|
|
# covers the `services` prefix, so a path outside it is refused rather
|
|
# than empty, however correct it reads.
|
|
name = "the collector's OIDC secret is read from the prefix the publisher writes";
|
|
ok =
|
|
let
|
|
s = otelBaoRemoteAuthelia.systemd.services.swarm-bao-otel-oidc.script;
|
|
in
|
|
lib.hasInfix "secret/swarm/services/swarm-collector/oidc/client" s
|
|
&& !(lib.hasInfix "secret/swarm/hives/" s);
|
|
}
|
|
{
|
|
# A copy on persistent disk outlives the secret's rotation and lands in
|
|
# backups of the container's root.
|
|
name = "the collector's OIDC secret is written to the host's /run only, and the old copy removed";
|
|
ok =
|
|
let
|
|
m = otelBaoRemoteAuthelia;
|
|
script = m.systemd.services.swarm-bao-otel-oidc.script;
|
|
mount = m.containers.swarm-otel.bindMounts."/run/swarm-otel-oidc";
|
|
rules = m.systemd.tmpfiles.rules;
|
|
in
|
|
lib.hasInfix "atomic_write_secret 0400 root:root /run/swarm-otel-oidc/swarm-collector.secret" script
|
|
&& !(lib.hasInfix "/var/lib/nixos-containers" script)
|
|
&& !(lib.hasInfix "/var/lib/swarm-otel-oidc" script)
|
|
&& mount.hostPath == "/run/swarm-otel-oidc"
|
|
&& mount.isReadOnly
|
|
&&
|
|
m.services.hyperhive.deploy.swarm-otel.clientSecretFile
|
|
== "/run/swarm-otel-oidc/swarm-collector.secret"
|
|
&& lib.elem "d /run/swarm-otel-oidc 0700 root root -" rules
|
|
&& lib.elem "f /run/swarm-otel-oidc/swarm-collector.secret 0400 root root -" rules
|
|
&& lib.elem "r /var/lib/nixos-containers/swarm-otel/var/lib/swarm-otel-oidc/swarm-collector.secret" rules
|
|
&&
|
|
lib.any (c: c == "oidc-client-secret:/run/swarm-otel-oidc/swarm-collector.secret")
|
|
m.containers.swarm-otel.config.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential;
|
|
}
|
|
{
|
|
# These exporters must not be gated on the stores' PER-HOST enables: a
|
|
# collector that did not share a host with them would render none at all
|
|
# and drop everything it received, from every hive — silently, because
|
|
# an absent exporter is not an error.
|
|
name = "a collector that hosts neither store still exports to both";
|
|
ok =
|
|
let
|
|
e = (otelSettings otelNoStores).exporters;
|
|
in
|
|
(e ? "otlphttp/victoriametrics") && (e ? "otlphttp/victorialogs");
|
|
}
|
|
{
|
|
# A swarm has one of each store, so the address is a swarm-level name.
|
|
# A loopback literal here is the co-location assumption written back in,
|
|
# and it renders, deploys and reports healthy while reaching nothing.
|
|
name = "the store exporters address the stores by name, never by loopback";
|
|
ok =
|
|
let
|
|
e = (otelSettings otelNoStores).exporters;
|
|
m = e."otlphttp/victoriametrics".metrics_endpoint;
|
|
l = e."otlphttp/victorialogs".logs_endpoint;
|
|
in
|
|
!(lib.hasInfix "127.0.0.1" m)
|
|
&& !(lib.hasInfix "127.0.0.1" l)
|
|
&& lib.hasInfix "metrics.t.local" m
|
|
&& lib.hasInfix "logs.t.local" l;
|
|
}
|
|
{
|
|
# `swarm.otel.audience` is built in ../host-modules/swarm-otel-service.nix,
|
|
# the push URLs the exporters use in ../host-modules/swarm-otel.nix. A URL
|
|
# the registration lacks is refused at the token endpoint with
|
|
# `invalid_target`, so every push fails while the collector reports
|
|
# healthy.
|
|
name = "the collector's registered audiences include both store push URLs it requests";
|
|
ok =
|
|
let
|
|
s = otelSettings otelBaoWithAuthelia;
|
|
registered = otelBaoWithAuthelia.services.hyperhive.swarm.otel.audience;
|
|
withoutQuery = url: lib.head (lib.splitString "?" url);
|
|
pushUrls = [
|
|
s.exporters."otlphttp/victoriametrics".metrics_endpoint
|
|
(withoutQuery s.exporters."otlphttp/victorialogs".logs_endpoint)
|
|
];
|
|
requested = map (n: s.extensions."oauth2client/${n}".endpoint_params.audience) [
|
|
"victoriametrics"
|
|
"victorialogs"
|
|
];
|
|
in
|
|
lib.all (u: builtins.elem u registered) pushUrls
|
|
&& lib.sort lib.lessThan requested == lib.sort lib.lessThan pushUrls;
|
|
}
|
|
{
|
|
# `_HOSTNAME` cannot separate machines on its own: a hostname is a
|
|
# config value two of them can share, and then every stream for a unit
|
|
# name merges into one.
|
|
name = "the log stream is keyed by machine, not only by a hostname every container shares";
|
|
ok =
|
|
let
|
|
l = (otelSettings otelNoStores).exporters."otlphttp/victorialogs".logs_endpoint;
|
|
field = f: lib.hasInfix ("_stream_fields=" + f) l || lib.hasInfix ("," + f) l;
|
|
in
|
|
field "_MACHINE_ID" && field "_SYSTEMD_UNIT" && !(field "_NOSUCHFIELD");
|
|
}
|
|
{
|
|
# Defining an exporter and REFERENCING it are two separate lists, and
|
|
# the second is where the original gate also lived. An exporter no
|
|
# pipeline names is as silent as one that does not exist — this case
|
|
# exists because a mutation that restored only the reference-side gate
|
|
# left every other case here green.
|
|
name = "every pipeline that has a store exporter defined actually sends to it";
|
|
ok =
|
|
let
|
|
s = otelSettings otelNoStores;
|
|
used = lib.unique (lib.concatMap (p: p.exporters) (lib.attrValues s.service.pipelines));
|
|
in
|
|
builtins.elem "otlphttp/victoriametrics" used && builtins.elem "otlphttp/victorialogs" used;
|
|
}
|
|
{
|
|
# An authenticator an exporter names but `service.extensions` omits is
|
|
# INERT — the collector starts clean and pushes unauthenticated until
|
|
# something at the far end refuses it. Checked as a set relation rather
|
|
# than by naming the two, so it keeps holding for exporters not written
|
|
# yet.
|
|
name = "every exporter authenticator is listed in service.extensions";
|
|
ok =
|
|
let
|
|
s = otelSettings otelNoStores;
|
|
named = lib.filter (v: v != null) (
|
|
lib.mapAttrsToList (_: e: e.auth.authenticator or null) s.exporters
|
|
);
|
|
in
|
|
named != [ ] && lib.all (a: builtins.elem a s.service.extensions) named;
|
|
}
|
|
{
|
|
# The host-journal sibling of ./agent-otel.nix's wiring case, which
|
|
# carries the full reasoning. Same question, different receiver: this
|
|
# one reads the HOST's journal rather than a container's, and the two
|
|
# are unrelated config — a fixed stanza there, a parameterised block
|
|
# inside `containers.swarm-otel` here — so one losing its parser while
|
|
# the other keeps one is a real and silent state.
|
|
#
|
|
# Contents are not this case's business. The table both receivers import
|
|
# is asserted once, in ./journald-severity.nix.
|
|
name = "the swarm collector's journald receiver carries the shared PRIORITY mapping";
|
|
ok = carriesJournaldSeverity (otelSettings otelNoStores).receivers.journald;
|
|
}
|
|
{
|
|
# Sibling of ./hive-otel.nix's identical case for the host-tier
|
|
# collector: this one's `oidc/*` extensions call out to authelia at
|
|
# startup, so it fails the same way when that restart races its own.
|
|
# nixpkgs ships `Restart = "always"` with no `RestartSec`, so without
|
|
# this the unit burns its five default attempts inside two seconds and
|
|
# lands in `start-limit-hit`, where it stops retrying. The third clause
|
|
# is the one that has to hold — `StartLimit*` are `[Unit]` settings
|
|
# that systemd ignores under `[Service]`, so a bound written into
|
|
# `serviceConfig` renders, deploys and does nothing.
|
|
name = "the swarm collector backs off a failed bind from [Unit], not [Service]";
|
|
ok =
|
|
let
|
|
u = otelNoStores.containers.swarm-otel.config.systemd.services.opentelemetry-collector;
|
|
in
|
|
u.serviceConfig.RestartSec or 0 > 0
|
|
&& toString u.unitConfig.StartLimitBurst == "12"
|
|
&& !(u.serviceConfig ? StartLimitBurst)
|
|
# The window has to outlast every attempt, or the burst is unreachable.
|
|
&& u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst;
|
|
}
|
|
{
|
|
# 🩸 The container's own journal reaches the stores from the collector
|
|
# itself. Mounting the host's journal over the guest's /var/log/journal
|
|
# (or `--link-journal=host`) leaves `journald/self` reading nothing while
|
|
# the collector looks healthy. Presence controls: the host receiver and
|
|
# its mount are asserted too, so a misspelt path cannot pass.
|
|
name = "the swarm collector ships its own journal, and the host's from a mount that leaves it alone";
|
|
ok =
|
|
let
|
|
c = otelNoStores.containers.swarm-otel;
|
|
s = otelSettings otelNoStores;
|
|
self = lib.filter (p: builtins.elem "journald/self" p.receivers) (
|
|
lib.attrValues s.service.pipelines
|
|
);
|
|
in
|
|
!(lib.elem "--link-journal=host" (c.extraFlags or [ ]))
|
|
&& s.receivers."journald/self".directory == "/var/log/journal"
|
|
&& s.receivers."journald/self".merge
|
|
&& s.receivers.journald.directory == "/run/host-journal"
|
|
&& c.bindMounts."/run/host-journal".hostPath == "/var/log/journal"
|
|
&& !(c.bindMounts ? "/var/log/journal")
|
|
&& lib.length self == 1
|
|
&& lib.all (p: p.exporters == [ "otlphttp/victorialogs" ]) self
|
|
&& lib.elem "resource/self-journal" (lib.head self).processors
|
|
&& builtins.elem "resource/self-journal" (lib.attrNames s.processors);
|
|
}
|
|
{
|
|
# 🩸 The collector logs its own export failures to the journal that
|
|
# `journald/self` reads. Shipping that to the upstream feeds each
|
|
# failure back into the failing exporter. Presence controls: the
|
|
# upstream exporter exists and the host-journal pipeline does send to
|
|
# it, so an exporter-name typo cannot make the negative pass.
|
|
name = "the collector's own journal never reaches the upstream, and drops the store exporter's own failures";
|
|
ok =
|
|
let
|
|
s = otelSettings otelNoStores;
|
|
pipelines = lib.attrValues s.service.pipelines;
|
|
self = lib.filter (p: builtins.elem "journald/self" p.receivers) pipelines;
|
|
host = lib.filter (p: builtins.elem "journald" p.receivers) pipelines;
|
|
f = s.processors."filter/drop-store-export-failures";
|
|
in
|
|
s.exporters ? otlphttp
|
|
&& lib.length host == 1
|
|
&& lib.all (p: builtins.elem "otlphttp" p.exporters) host
|
|
&& lib.length self == 1
|
|
&& !(lib.any (p: builtins.elem "otlphttp" p.exporters) self)
|
|
&& lib.elem "filter/drop-store-export-failures" (lib.head self).processors
|
|
&& lib.length f.logs.log_record == 1
|
|
&& lib.hasInfix "Exporting failed" (lib.head f.logs.log_record)
|
|
&& lib.hasInfix "otlphttp/victorialogs" (lib.head f.logs.log_record);
|
|
}
|
|
{
|
|
# The journald receiver saves its cursor before an entry reaches the
|
|
# exporter, so an entry in an in-memory queue when the store is down or
|
|
# the collector restarts is gone for good. The queue has to sit on the
|
|
# `file_storage` extension, and the retry has to outlast a switch.
|
|
# Presence controls: the extension is defined and started, the queue is
|
|
# bounded, and the exporters carry their endpoints.
|
|
name = "the log exporters queue on disk and retry without a time limit";
|
|
ok =
|
|
let
|
|
s = otelSettings otelNoStores;
|
|
queued =
|
|
e:
|
|
e.sending_queue.enabled
|
|
&& e.sending_queue.storage == "file_storage"
|
|
&& e.sending_queue.queue_size > 0
|
|
&& e.retry_on_failure.enabled
|
|
&& e.retry_on_failure.max_elapsed_time == "0s";
|
|
in
|
|
queued s.exporters."otlphttp/victorialogs"
|
|
&& queued s.exporters.otlphttp
|
|
&& s.exporters."otlphttp/victorialogs" ? logs_endpoint
|
|
&& s.exporters.otlphttp.endpoint == "https://upstream.example/otel"
|
|
&& s.extensions ? file_storage
|
|
&& builtins.elem "file_storage" s.service.extensions;
|
|
}
|
|
{
|
|
# Presence control: the default sends both signals to the upstream, so
|
|
# the negative arms below cannot pass on a misspelled exporter name.
|
|
name = "the upstream receives metrics and logs by default";
|
|
ok =
|
|
let
|
|
s = otelSettings otelNoStores;
|
|
ps = s.service.pipelines;
|
|
names = pre: lib.filter (lib.hasPrefix pre) (lib.attrNames ps);
|
|
in
|
|
s.exporters ? otlphttp
|
|
&& names "metrics/" != [ ]
|
|
&& names "logs/" != [ ]
|
|
&& lib.all (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "metrics/")
|
|
&& lib.all (n: n == "logs/self" || builtins.elem "otlphttp" ps.${n}.exporters) (names "logs/");
|
|
}
|
|
{
|
|
name = "upstreamSignals limits which pipelines export upstream, and the stores keep everything";
|
|
ok =
|
|
let
|
|
s = otelSettings otelUpstreamMetricsOnly;
|
|
ps = s.service.pipelines;
|
|
names = pre: lib.filter (lib.hasPrefix pre) (lib.attrNames ps);
|
|
in
|
|
s.exporters ? otlphttp
|
|
&& names "metrics/" != [ ]
|
|
&& names "logs/" != [ ]
|
|
&& lib.all (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "metrics/")
|
|
&& lib.all (n: builtins.elem "otlphttp/victoriametrics" ps.${n}.exporters) (names "metrics/")
|
|
&& !(lib.any (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "logs/"))
|
|
&& lib.all (n: builtins.elem "otlphttp/victorialogs" ps.${n}.exporters) (names "logs/");
|
|
}
|
|
{
|
|
# The control is `otelNoStores`: it sets the new paths and warns about
|
|
# none of the old ones, so the warnings below come from the renames.
|
|
name = "the old otel.* upstream paths set deploy.swarm-otel.upstream and warn";
|
|
ok =
|
|
let
|
|
u = otelOldUpstreamPaths.services.hyperhive.deploy.swarm-otel.upstream;
|
|
warnsAbout = m: path: lib.any (lib.hasInfix "services.hyperhive.${path}") m.warnings;
|
|
in
|
|
u.endpoint == "https://old.example/otel"
|
|
&& u.protocol == "http/json"
|
|
&& u.headersCredential == "/run/secrets/old-otel-headers"
|
|
&& u.headerName == "X-Old-Token"
|
|
&& warnsAbout otelOldUpstreamPaths "otel.endpoint"
|
|
&& warnsAbout otelOldUpstreamPaths "otel.protocol"
|
|
&& warnsAbout otelOldUpstreamPaths "otel.headersCredential"
|
|
&& warnsAbout otelOldUpstreamPaths "otel.collector.upstreamHeaderName"
|
|
&& !(warnsAbout otelNoStores "otel.endpoint")
|
|
&&
|
|
otelNoStores.services.hyperhive.deploy.swarm-otel.upstream.endpoint
|
|
== "https://upstream.example/otel"
|
|
&& (otelSettings otelOldUpstreamPaths).exporters ? otlphttp;
|
|
}
|
|
{
|
|
# The collector rejects an exporter no pipeline uses, so with nothing
|
|
# going upstream the exporter must not be defined.
|
|
name = "an empty upstream.signals defines no upstream exporter and keeps the stores";
|
|
ok =
|
|
let
|
|
s = otelSettings otelUpstreamNone;
|
|
ps = lib.attrValues s.service.pipelines;
|
|
in
|
|
!(s.exporters ? otlphttp)
|
|
&& !(lib.any (p: builtins.elem "otlphttp" p.exporters) ps)
|
|
&& s.exporters ? "otlphttp/victoriametrics"
|
|
&& s.exporters ? "otlphttp/victorialogs"
|
|
&& lib.any (p: builtins.elem "otlphttp/victorialogs" p.exporters) ps
|
|
&& lib.any (p: builtins.elem "otlphttp/victoriametrics" p.exporters) ps;
|
|
}
|
|
];
|
|
in
|
|
runGroup "swarm-otel-core" cases
|