dc418a5223
The single module-eval derivation forced ~62 full nixosSystem fixtures live at once to compute its cases list: 10.6GB peak RSS / 5m25s to evaluate, by far the dominant cost in nix flake check. Splits it into 21 independent checks.module-eval-* derivations (1-7 fixtures each) sharing builders/helpers via module-eval/lib.nix, so no single derivation needs more than a handful of fixtures live at once. A few cases spanning two clusters carry a small duplicated fixture rather than threading shared state through lib.nix.
160 lines
6.4 KiB
Nix
160 lines
6.4 KiB
Nix
# `checks.module-eval-swarm-otel-core` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
runGroup
|
|
otelSettings
|
|
;
|
|
|
|
# A swarm collector on a host that runs NEITHER store — the fully-spread
|
|
# shape from docs/swarm/services.md, and the one the old per-host gates made
|
|
# inexpressible. It is the whole point of the cases below that this hive is
|
|
# not a degenerate configuration but a supported one.
|
|
otelNoStores = hive {
|
|
deploy.swarm-otel.enable = true;
|
|
deploy.authelia.enable = true;
|
|
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
|
|
deploy.victoriametrics.enable = false;
|
|
deploy.victorialogs.enable = false;
|
|
};
|
|
|
|
# The collector beside authelia, reading its own OIDC secret out of the
|
|
# store like every other collector — the cert pair here is not scenery, it
|
|
# is the arm that would catch the deleted co-located copy unit coming back.
|
|
otelBaoWithAuthelia = hive {
|
|
deploy.swarm-otel.enable = true;
|
|
deploy.authelia.enable = true;
|
|
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
|
|
};
|
|
|
|
# The same collector with the IdP on ANOTHER host and a store leaf placed by
|
|
# hand. Identical to the fixture above in everything the delivery path
|
|
# reads, which is the point.
|
|
otelBaoRemoteAuthelia = hive {
|
|
deploy.swarm-otel.enable = true;
|
|
swarm.authelia.url = "https://auth.example.invalid";
|
|
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
|
|
};
|
|
cases = [
|
|
{
|
|
# 🩸 The arm that guards the ruling this slice landed under, the
|
|
# collector's half of ./swarm-grafana.nix's own. There is ONE delivery
|
|
# route: the store reader, on every host that runs the collector and
|
|
# holds a store identity. The negative names the deleted unit rather
|
|
# than a generic absence, because the way this regresses is someone
|
|
# re-adding the co-located copy as an optimisation.
|
|
name = "the collector's OIDC secret has exactly one delivery unit, the store reader, in both topologies";
|
|
ok =
|
|
let
|
|
local = otelBaoWithAuthelia.systemd.services;
|
|
remote = otelBaoRemoteAuthelia.systemd.services;
|
|
in
|
|
local ? swarm-bao-otel-oidc
|
|
&& remote ? swarm-bao-otel-oidc
|
|
&& !(local ? swarm-otel-oidc-secret)
|
|
&& !(remote ? swarm-otel-oidc-secret);
|
|
}
|
|
{
|
|
# Same 403-not-a-miss reason as grafana's arm above: the reader's grant
|
|
# covers the `services` prefix, so a path outside it is refused rather
|
|
# than empty, however correct it reads.
|
|
name = "the collector's OIDC secret is read from the prefix the publisher writes";
|
|
ok =
|
|
let
|
|
s = otelBaoRemoteAuthelia.systemd.services.swarm-bao-otel-oidc.script;
|
|
in
|
|
lib.hasInfix "secret/swarm/services/swarm-collector/oidc/client" s
|
|
&& !(lib.hasInfix "secret/swarm/hives/" s);
|
|
}
|
|
{
|
|
# The defect itself. These exporters used to be gated on the stores'
|
|
# PER-HOST enables, so a collector that did not share a host with them
|
|
# rendered none at all and dropped everything it received, from every
|
|
# hive — silently, because an absent exporter is not an error.
|
|
name = "a collector that hosts neither store still exports to both";
|
|
ok =
|
|
let
|
|
e = (otelSettings otelNoStores).exporters;
|
|
in
|
|
(e ? "otlphttp/victoriametrics") && (e ? "otlphttp/victorialogs");
|
|
}
|
|
{
|
|
# A swarm has one of each store, so the address is a swarm-level name.
|
|
# A loopback literal here is the co-location assumption written back in,
|
|
# and it renders, deploys and reports healthy while reaching nothing.
|
|
name = "the store exporters address the stores by name, never by loopback";
|
|
ok =
|
|
let
|
|
e = (otelSettings otelNoStores).exporters;
|
|
m = e."otlphttp/victoriametrics".metrics_endpoint;
|
|
l = e."otlphttp/victorialogs".logs_endpoint;
|
|
in
|
|
!(lib.hasInfix "127.0.0.1" m)
|
|
&& !(lib.hasInfix "127.0.0.1" l)
|
|
&& lib.hasInfix "metrics.t.local" m
|
|
&& lib.hasInfix "logs.t.local" l;
|
|
}
|
|
{
|
|
# `_HOSTNAME` cannot separate machines on its own: a hostname is a
|
|
# config value two of them can share, and then every stream for a unit
|
|
# name merges into one.
|
|
name = "the log stream is keyed by machine, not only by a hostname every container shares";
|
|
ok =
|
|
let
|
|
l = (otelSettings otelNoStores).exporters."otlphttp/victorialogs".logs_endpoint;
|
|
field = f: lib.hasInfix ("_stream_fields=" + f) l || lib.hasInfix ("," + f) l;
|
|
in
|
|
field "_MACHINE_ID" && field "_SYSTEMD_UNIT" && !(field "_NOSUCHFIELD");
|
|
}
|
|
{
|
|
# Defining an exporter and REFERENCING it are two separate lists, and
|
|
# the second is where the original gate also lived. An exporter no
|
|
# pipeline names is as silent as one that does not exist — this case
|
|
# exists because a mutation that restored only the reference-side gate
|
|
# left every other case here green.
|
|
name = "every pipeline that has a store exporter defined actually sends to it";
|
|
ok =
|
|
let
|
|
s = otelSettings otelNoStores;
|
|
used = lib.unique (lib.concatMap (p: p.exporters) (lib.attrValues s.service.pipelines));
|
|
in
|
|
builtins.elem "otlphttp/victoriametrics" used && builtins.elem "otlphttp/victorialogs" used;
|
|
}
|
|
{
|
|
# An authenticator an exporter names but `service.extensions` omits is
|
|
# INERT — the collector starts clean and pushes unauthenticated until
|
|
# something at the far end refuses it. Checked as a set relation rather
|
|
# than by naming the two, so it keeps holding for exporters not written
|
|
# yet.
|
|
name = "every exporter authenticator is listed in service.extensions";
|
|
ok =
|
|
let
|
|
s = otelSettings otelNoStores;
|
|
named = lib.filter (v: v != null) (
|
|
lib.mapAttrsToList (_: e: e.auth.authenticator or null) s.exporters
|
|
);
|
|
in
|
|
named != [ ] && lib.all (a: builtins.elem a s.service.extensions) named;
|
|
}
|
|
];
|
|
in
|
|
runGroup "swarm-otel-core" cases
|