e4e62be506
On a first boot the secret file does not exist until the first fetch succeeds, so the collector fails `LoadCredential` into its start limit, which leaves it `failed`. `refresh_unit` already handles that: it sees `is-failed`, runs `reset-failed` (which clears the start-limit counter) and starts it. This case pins that arm, the empty-ActiveEnterTimestamp fallback that makes the file count as newer, and that the collector only wants the fetch rather than requiring it. Refs #5015
311 lines
14 KiB
Nix
311 lines
14 KiB
Nix
# `checks.module-eval-hive-otel` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
runGroup
|
|
otelSettings
|
|
;
|
|
|
|
# This hive's own collector, which is a HOST service — unlike the swarm
|
|
# tier's, which lives in a container and is read through `otelSettings`.
|
|
hiveOtel = hive {
|
|
otel.enable = true;
|
|
otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret";
|
|
};
|
|
|
|
# A hive holding its own store leaf, placed by hand, and naming no secret
|
|
# file: the collector's secret comes from the store under that leaf.
|
|
hiveOtelLeaf = hive {
|
|
otel.enable = true;
|
|
deploy.bao.clientCertFile = "/etc/pki/hive.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/hive-key.pem";
|
|
};
|
|
|
|
# The IdP, the store and the collector on one host: the publisher that
|
|
# writes the secret and the unit that reads it, in one config.
|
|
hiveOtelAllLocal = hive {
|
|
otel.enable = true;
|
|
deploy.bao.enable = true;
|
|
deploy.authelia.enable = true;
|
|
};
|
|
|
|
hiveOtelFetch = machine: machine.systemd.services.swarm-bao-hive-otel-oidc;
|
|
|
|
hiveOtelPipelines = hiveOtel.services.opentelemetry-collector.settings.service.pipelines;
|
|
|
|
hiveOtelSettings = hiveOtel.services.opentelemetry-collector.settings;
|
|
|
|
# The hive tier's rendered scrape list. Same reasoning as `scrapeJob` for
|
|
# the swarm tier — the option is one string, what prometheus is handed is a
|
|
# job — but this collector is a host service, so the path to it differs.
|
|
hiveScrapeJob =
|
|
job:
|
|
lib.findFirst (
|
|
c: c.job_name == job
|
|
) null hiveOtelSettings.receivers.prometheus.config.scrape_configs;
|
|
# Duplicated from bao-otel-collector.nix — a case here needs it too.
|
|
|
|
# The store with and without a collector on the same host. `scrapeTargets`
|
|
# is only ever read by a local collector, so the metrics endpoint is a
|
|
# function of the pairing rather than of the store.
|
|
baoWithCollector = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.swarm-otel.enable = true;
|
|
# A second job declared as bare `host:port`, so the pair of cases below
|
|
# reads one rendered scrape list: the store's entry carries a path, this
|
|
# one carries none.
|
|
swarm.otel.scrapeTargets.plain = "127.0.0.1:9999";
|
|
};
|
|
|
|
# Also duplicated from bao-otel-collector.nix, where it is the fixture for
|
|
# the store standing beside a HIVE collector. That co-location is exactly
|
|
# the topology the port case below needs: two collectors, one netns, and
|
|
# nothing but their self-telemetry ports keeping them apart.
|
|
# `clientSecretFile` is what ../host-modules/otel.nix's identity assertion
|
|
# demands of any hive with the tier on.
|
|
baoWithHiveOtel = hive {
|
|
deploy.bao.enable = true;
|
|
otel.enable = true;
|
|
otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret";
|
|
};
|
|
|
|
# The collector INSIDE the store's container — neither the host's nor the
|
|
# swarm tier's.
|
|
baoForwarderSettings =
|
|
machine: machine.containers.swarm-bao.config.services.opentelemetry-collector.settings;
|
|
cases = [
|
|
{
|
|
# The tier in the middle. Its OTLP receiver takes both signals on one
|
|
# port, so without this pipeline an agent's push is answered 404 on
|
|
# `/v1/logs` — and a forwarder retrying into a 404 is indistinguish-
|
|
# able from one with nothing to send. Compared against the metrics
|
|
# pipeline's exporters rather than a name, so the two signals cannot
|
|
# drift to different destinations.
|
|
name = "the hive collector forwards logs upstream, not only metrics";
|
|
ok =
|
|
(hiveOtelPipelines ? logs)
|
|
&& hiveOtelPipelines.metrics.exporters != [ ]
|
|
&& hiveOtelPipelines.logs.receivers == [ "otlp" ]
|
|
&& hiveOtelPipelines.logs.exporters == hiveOtelPipelines.metrics.exporters;
|
|
}
|
|
{
|
|
# `deltatocumulative` is metrics-only: naming it in a logs pipeline
|
|
# kills the collector at startup rather than doing nothing. The second
|
|
# clause is the control — the metrics pipeline still names it, so a
|
|
# pass means the two processor lists differ rather than that the
|
|
# processor left the module.
|
|
name = "the hive collector keeps the metrics-only processor out of its logs pipeline";
|
|
ok =
|
|
!(builtins.elem "deltatocumulative" hiveOtelPipelines.logs.processors)
|
|
&& builtins.elem "deltatocumulative" hiveOtelPipelines.metrics.processors;
|
|
}
|
|
{
|
|
# The counters that say telemetry is being LOST — refused, failed,
|
|
# queue depth — are served on loopback and reach no store unless
|
|
# something reads them. Read off the rendered job rather than the
|
|
# option: only the job is what prometheus actually requests.
|
|
name = "the hive collector scrapes its own telemetry endpoint";
|
|
ok =
|
|
let
|
|
j = hiveScrapeJob "collector";
|
|
in
|
|
j != null && j.static_configs == [ { targets = [ "127.0.0.1:8888" ]; } ];
|
|
}
|
|
{
|
|
# A `prometheus` receiver no pipeline names collects nothing while
|
|
# rendering and starting perfectly, so the scrape above is inert
|
|
# without this. The path is newly reachable: until the hive tier had a
|
|
# target of its own, this receiver was never emitted on any hive.
|
|
name = "the hive metrics pipeline names the prometheus receiver the self-scrape needs";
|
|
ok = builtins.elem "prometheus" hiveOtelPipelines.metrics.receivers;
|
|
}
|
|
{
|
|
# `metrics.address` is the spelling that looks right and is rejected by
|
|
# this collector version. The first clause is the control: without it a
|
|
# missing telemetry block would pass the port check vacuously.
|
|
name = "the hive collector binds its telemetry port through readers, not address";
|
|
ok =
|
|
let
|
|
m = hiveOtelSettings.service.telemetry.metrics;
|
|
in
|
|
!(m ? address) && (lib.head m.readers).pull.exporter.prometheus.port == 8888;
|
|
}
|
|
{
|
|
# A bind that loses the race with the outgoing process is the whole
|
|
# failure: nixpkgs ships `Restart = "always"` with no `RestartSec`, so
|
|
# without this the unit spends its five default attempts inside two
|
|
# seconds and lands in `start-limit-hit`, where it stops retrying. The
|
|
# third clause is the one that has to hold — `StartLimit*` are `[Unit]`
|
|
# settings that systemd ignores under `[Service]`, so a bound written
|
|
# into `serviceConfig` renders, deploys and does nothing. Asserted
|
|
# where nixpkgs puts it rather than where it was written, same as
|
|
# ./bao-grants.nix.
|
|
name = "the hive collector backs off a failed bind from [Unit], not [Service]";
|
|
ok =
|
|
let
|
|
u = hiveOtel.systemd.services.opentelemetry-collector;
|
|
in
|
|
u.serviceConfig.RestartSec or 0 > 0
|
|
&& toString u.unitConfig.StartLimitBurst == "12"
|
|
&& !(u.serviceConfig ? StartLimitBurst)
|
|
# The window has to outlast every attempt, or the burst is unreachable.
|
|
&& u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst;
|
|
}
|
|
{
|
|
# Collectors share a network namespace whenever they are co-located,
|
|
# and this port appears in no config the port-collision assertion can
|
|
# read — so equal defaults mean the second to start dies at `bind()`.
|
|
# Pinned as a case rather than an assertion: enforcing it belongs with
|
|
# the other port checks, not here.
|
|
#
|
|
# THREE tiers, because the store's container grew a collector of its
|
|
# own and this case could not see it: an UNDECLARED port is not an
|
|
# absent one, it is the binary's 8888, which is the hive tier's. That
|
|
# is why `portOf` falls back to 8888 rather than throwing — a tier that
|
|
# declares nothing has to fail this case loudly, not evaluate away.
|
|
name = "no two collector tiers claim the same self-telemetry port";
|
|
ok =
|
|
let
|
|
portOf =
|
|
s:
|
|
(lib.head (s.service.telemetry.metrics.readers or [ { } ])).pull.exporter.prometheus.port or 8888;
|
|
hivePort = portOf hiveOtelSettings;
|
|
swarmPort = portOf (otelSettings baoWithCollector);
|
|
baoPort = portOf (baoForwarderSettings baoWithHiveOtel);
|
|
in
|
|
hivePort != swarmPort && hivePort != baoPort && swarmPort != baoPort;
|
|
}
|
|
{
|
|
# The collector authenticates as the hive, so it reads its secret as
|
|
# the hive: the leaf `deploy.bao.clientCertFile` names, whose policy
|
|
# grants `swarm/hives/<hive>/*`. The last two clauses tie the unit's
|
|
# output to what the collector loads.
|
|
name = "the hive collector's secret is fetched from the store under the hive's own leaf";
|
|
ok =
|
|
let
|
|
u = hiveOtelFetch hiveOtelLeaf;
|
|
b = hiveOtelLeaf.services.hyperhive.deploy.bao;
|
|
file = hiveOtelLeaf.services.hyperhive.otel.clientSecretFile;
|
|
s = builtins.unsafeDiscardStringContext u.script;
|
|
in
|
|
hiveOtelLeaf.systemd.services ? swarm-bao-hive-otel-oidc
|
|
&& u.environment.BAO_CLIENT_CERT == b.clientCertFile
|
|
&& u.environment.BAO_CLIENT_KEY == b.clientKeyFile
|
|
&& lib.hasInfix "bao login -method=cert" s
|
|
&& lib.hasInfix "bao kv get -field=value secret/swarm/hives/h1/oidc/client" s
|
|
&& lib.hasInfix "atomic_write_secret 0400 \"\" ${file} " s
|
|
&& builtins.elem "swarm-client.secret:${file}" hiveOtelLeaf.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
|
|
&& builtins.elem "opentelemetry-collector.service" u.wantedBy
|
|
&& builtins.elem "opentelemetry-collector.service" u.before;
|
|
}
|
|
{
|
|
# `LoadCredential` copies the file at the collector's start, so a value
|
|
# landed later reaches it only through a restart. Retries every 30s
|
|
# (./lib/store-retry.nix) while the store is down or the secret
|
|
# unpublished.
|
|
name = "the hive collector's fetch retries and restarts the collector when the file is newer";
|
|
ok =
|
|
let
|
|
u = hiveOtelFetch hiveOtelLeaf;
|
|
s = builtins.unsafeDiscardStringContext u.script;
|
|
in
|
|
u.serviceConfig.Restart == "on-failure"
|
|
&& u.serviceConfig.RestartSec == 30
|
|
&& toString u.unitConfig.StartLimitBurst == "2880"
|
|
&& lib.hasInfix "refresh_unit opentelemetry-collector.service /var/lib/hive-otel-oidc/hive-h1.secret" s
|
|
&& lib.hasInfix "ActiveEnterTimestamp" s
|
|
&& lib.hasInfix ''if [ "$written_us" -le "$started_us" ]; then'' s;
|
|
}
|
|
{
|
|
# On a first boot the file does not exist until the fetch succeeds, so
|
|
# the collector fails `LoadCredential` into its start limit, i.e. the
|
|
# `failed` state. Only `reset-failed` clears that limit, so the fetch
|
|
# has to reset before it starts; `try-restart` acts on no failed unit.
|
|
# A never-active collector has an empty `ActiveEnterTimestamp`, read as
|
|
# 0, so the file always counts as newer. The collector only `wants` the
|
|
# fetch, so an unreachable store does not hold it back.
|
|
name = "the hive collector's fetch revives a collector that hit its start limit without the file";
|
|
ok =
|
|
let
|
|
s = builtins.unsafeDiscardStringContext (hiveOtelFetch hiveOtelLeaf).script;
|
|
c = hiveOtelLeaf.systemd.services.opentelemetry-collector;
|
|
failedArm = lib.last (lib.splitString ''is-failed --quiet "$unit"; then'' s);
|
|
in
|
|
lib.hasInfix ''is-failed --quiet "$unit"; then'' s
|
|
&& lib.hasInfix ''reset-failed "$unit"'' failedArm
|
|
&& lib.hasInfix ''start --no-block "$unit"'' (
|
|
lib.last (lib.splitString ''reset-failed "$unit"'' failedArm)
|
|
)
|
|
&& lib.hasInfix "started_us=0" s
|
|
&& !(builtins.elem "swarm-bao-hive-otel-oidc.service" (c.requires or [ ]));
|
|
}
|
|
{
|
|
# The store is the one route, on the IdP's own host too: no default into
|
|
# authelia's tree and no unit waiting for a file to appear there.
|
|
name = "the hive collector reads no secret out of authelia's tree and waits on no copied file";
|
|
ok =
|
|
let
|
|
file = hiveOtelAllLocal.services.hyperhive.otel.clientSecretFile;
|
|
c = hiveOtelAllLocal.systemd.services.opentelemetry-collector;
|
|
in
|
|
!(hiveOtelAllLocal.systemd.services ? hive-otel-client-secret)
|
|
&& !(hiveOtelLeaf.systemd.services ? hive-otel-client-secret)
|
|
&& !(lib.hasInfix "authelia" file)
|
|
&& hiveOtelAllLocal.systemd.services ? swarm-bao-hive-otel-oidc
|
|
&& !(builtins.elem "hive-otel-client-secret.service" (c.requires or [ ]));
|
|
}
|
|
{
|
|
# The writer and the reader spell the path independently; this is the
|
|
# one place that sees both rendered.
|
|
name = "the hive collector reads the path the secret publisher writes";
|
|
ok =
|
|
let
|
|
fetch = builtins.unsafeDiscardStringContext (hiveOtelFetch hiveOtelAllLocal).script;
|
|
publish = builtins.unsafeDiscardStringContext hiveOtelAllLocal.systemd.services.swarm-secret-publish.script;
|
|
path = "secret/swarm/hives/h1/oidc/client";
|
|
in
|
|
lib.hasInfix "bao kv get -field=value ${path}" fetch && lib.hasInfix "bao kv put ${path} " publish;
|
|
}
|
|
{
|
|
# A secret delivered some other way is the operator's: no unit fetches
|
|
# into a file the collector does not load, or restarts it on one. Nothing
|
|
# restarts the collector when that file lands, so it waits for the file
|
|
# instead of failing `LoadCredential` into its start limit.
|
|
name = "a hand-named collector secret renders no store fetch, and the collector waits for the file";
|
|
ok =
|
|
let
|
|
waitsFor =
|
|
machine:
|
|
let
|
|
c = machine.systemd.services.opentelemetry-collector;
|
|
w = machine.systemd.services.hive-otel-client-secret;
|
|
in
|
|
builtins.elem "hive-otel-client-secret.service" c.requires
|
|
&& builtins.elem "hive-otel-client-secret.service" c.after
|
|
&& lib.hasInfix "/var/lib/hive-otel-oidc/client.secret" w.script;
|
|
in
|
|
!(hiveOtel.systemd.services ? swarm-bao-hive-otel-oidc)
|
|
&& !(baoWithHiveOtel.systemd.services ? swarm-bao-hive-otel-oidc)
|
|
&& builtins.elem "swarm-client.secret:/var/lib/hive-otel-oidc/client.secret" baoWithHiveOtel.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
|
|
&& waitsFor hiveOtel
|
|
&& waitsFor baoWithHiveOtel;
|
|
}
|
|
];
|
|
in
|
|
runGroup "hive-otel" cases
|