b68a8ef290
swarm-secret-publish and its path unit now run inside authelia's container, beside the mint, and read each client secret at the path authelia writes it. The host unit and host path unit, which read the secrets through the container's root on the host, are gone, and so is deploy.authelia.hostClientSecretDir, which only they read. The publisher keeps its store identity, the secret-publisher leaf. The host hands the leaf and the store's CA into the container as nspawn system credentials, and the unit loads them; where this host mints the leaf, the container waits for swarm-bao-pki. Authelia does not wait for the publisher, which retries every 30 seconds while the store is down. The container also runs the journal forwarder the other service containers run, pushing to a receiver of its own on the swarm collector (serviceProducers swarm-authelia) as a new machine OIDC client (swarm.authelia.otel.clientId) that authelia registers for itself and the publisher carries to the store. The fetch unit reads it back under the publisher's leaf, also handed to it as a system credential, so the swarm-secret-publisher policy gains read on that one leaf; its stanza keeps create and update, which the exact path would otherwise take from the services/* prefix. Without the leaf the forwarder is off. The container drops --link-journal=host, and the host collector stops reading it. The services' "nothing at <path>" errors name the publisher in authelia's container.
374 lines
18 KiB
Nix
374 lines
18 KiB
Nix
# `checks.module-eval-hive-otel` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
shared,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
runGroup
|
|
otelSettings
|
|
;
|
|
|
|
# This hive's own collector, which is a HOST service — unlike the swarm
|
|
# tier's, which lives in a container and is read through `otelSettings`.
|
|
# The IdP is here too, with a hand-named secret file: the collector's wait
|
|
# for that file exists beside authelia's container.
|
|
hiveOtel = hive {
|
|
otel.enable = true;
|
|
otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret";
|
|
deploy.authelia.enable = true;
|
|
};
|
|
|
|
# A hive holding its own store leaf, placed by hand, and naming no secret
|
|
# file: the collector's secret comes from the store under that leaf.
|
|
hiveOtelLeaf = hive {
|
|
otel.enable = true;
|
|
deploy.bao.clientCertFile = "/etc/pki/hive.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/hive-key.pem";
|
|
};
|
|
|
|
# The IdP, the store and the collector on one host: the publisher that
|
|
# writes the secret and the unit that reads it, in one config.
|
|
hiveOtelAllLocal = hive {
|
|
otel.enable = true;
|
|
deploy.bao.enable = true;
|
|
deploy.authelia.enable = true;
|
|
};
|
|
|
|
hiveOtelFetch = machine: machine.systemd.services.swarm-bao-hive-otel-oidc;
|
|
|
|
hiveOtelPipelines = hiveOtel.services.opentelemetry-collector.settings.service.pipelines;
|
|
|
|
hiveOtelSettings = hiveOtel.services.opentelemetry-collector.settings;
|
|
|
|
# The hive tier's rendered scrape list. Same reasoning as `scrapeJob` for
|
|
# the swarm tier — the option is one string, what prometheus is handed is a
|
|
# job — but this collector is a host service, so the path to it differs.
|
|
hiveScrapeJob =
|
|
job:
|
|
lib.findFirst (
|
|
c: c.job_name == job
|
|
) null hiveOtelSettings.receivers.prometheus.config.scrape_configs;
|
|
|
|
# The store with and without a collector on the same host. `scrapeTargets`
|
|
# is only ever read by a local collector, so the metrics endpoint is a
|
|
# function of the pairing rather than of the store.
|
|
inherit (shared) baoWithCollector;
|
|
|
|
# The store standing beside a HIVE collector: the topology the port case
|
|
# below needs, two collectors in one netns with nothing but their
|
|
# self-telemetry ports keeping them apart.
|
|
inherit (shared) baoWithHiveOtel;
|
|
|
|
# The collector INSIDE the store's container — neither the host's nor the
|
|
# swarm tier's.
|
|
baoForwarderSettings =
|
|
machine: machine.containers.swarm-bao.config.services.opentelemetry-collector.settings;
|
|
cases = [
|
|
{
|
|
# The tier in the middle. Its OTLP receiver takes both signals on one
|
|
# port, so without this pipeline an agent's push is answered 404 on
|
|
# `/v1/logs` — and a forwarder retrying into a 404 is indistinguish-
|
|
# able from one with nothing to send. `journald` joined the receiver
|
|
# list for this collector's own host journal (hive-c0re's crash-watch
|
|
# log line, see ./otel.nix) — both have to be present, in either
|
|
# order, so this stays a set check rather than a list-equality one.
|
|
# Compared against the metrics pipeline's exporters rather than a
|
|
# name, so the two signals cannot drift to different destinations.
|
|
name = "the hive collector forwards logs upstream, not only metrics";
|
|
ok =
|
|
(hiveOtelPipelines ? logs)
|
|
&& hiveOtelPipelines.metrics.exporters != [ ]
|
|
&&
|
|
lib.sort builtins.lessThan hiveOtelPipelines.logs.receivers == [
|
|
"journald"
|
|
"otlp"
|
|
]
|
|
&& hiveOtelPipelines.logs.exporters == hiveOtelPipelines.metrics.exporters;
|
|
}
|
|
{
|
|
# `deltatocumulative` is metrics-only: naming it in a logs pipeline
|
|
# kills the collector at startup rather than doing nothing. The second
|
|
# clause is the control — the metrics pipeline still names it, so a
|
|
# pass means the two processor lists differ rather than that the
|
|
# processor left the module.
|
|
name = "the hive collector keeps the metrics-only processor out of its logs pipeline";
|
|
ok =
|
|
!(builtins.elem "deltatocumulative" hiveOtelPipelines.logs.processors)
|
|
&& builtins.elem "deltatocumulative" hiveOtelPipelines.metrics.processors;
|
|
}
|
|
{
|
|
# `otelcol validate` refuses a `service.extensions` entry with no
|
|
# definition, and that runs when the unit is built, not in this suite.
|
|
# `hiveOtel` authenticates upstream, so both extensions are listed;
|
|
# the length clause keeps the check from passing on an empty list.
|
|
name = "the hive collector defines every extension its service lists";
|
|
ok =
|
|
lib.length hiveOtelSettings.service.extensions == 2
|
|
&& lib.all (e: hiveOtelSettings.extensions ? ${e}) hiveOtelSettings.service.extensions;
|
|
}
|
|
{
|
|
# The counters that say telemetry is being LOST — refused, failed,
|
|
# queue depth — are served on loopback and reach no store unless
|
|
# something reads them. Read off the rendered job rather than the
|
|
# option: only the job is what prometheus actually requests.
|
|
name = "the hive collector scrapes its own telemetry endpoint";
|
|
ok =
|
|
let
|
|
j = hiveScrapeJob "collector";
|
|
in
|
|
j != null && j.static_configs == [ { targets = [ "127.0.0.1:8888" ]; } ];
|
|
}
|
|
{
|
|
# A `prometheus` receiver no pipeline names collects nothing while
|
|
# rendering and starting perfectly, so the scrape above is inert
|
|
# without this. The path is newly reachable: until the hive tier had a
|
|
# target of its own, this receiver was never emitted on any hive.
|
|
name = "the hive metrics pipeline names the prometheus receiver the self-scrape needs";
|
|
ok = builtins.elem "prometheus" hiveOtelPipelines.metrics.receivers;
|
|
}
|
|
{
|
|
# The receiver, the pipeline entry and the nginx server it reads have to
|
|
# agree on one address, and none of them may exist without a gateway on
|
|
# the host. `hiveOtelLeaf` is the control: it runs no gateway, and the
|
|
# clause asserting that keeps the control from going vacuous.
|
|
name = "the hive collector reads the gateway's stub_status, and only where a gateway runs";
|
|
ok =
|
|
let
|
|
nginxConf = hiveOtel.services.nginx.appendHttpConfig;
|
|
leafSettings = hiveOtelLeaf.services.opentelemetry-collector.settings;
|
|
in
|
|
hiveOtel.services.hyperhive.gateway.enable
|
|
&& hiveOtelSettings.receivers.nginx.endpoint == "http://127.0.0.1:8085/stub_status"
|
|
&& builtins.elem "nginx" hiveOtelPipelines.metrics.receivers
|
|
&& lib.hasInfix "listen 127.0.0.1:8085;" nginxConf
|
|
&& lib.hasInfix "location = /stub_status {" nginxConf
|
|
&& !hiveOtelLeaf.services.hyperhive.gateway.enable
|
|
&& !(leafSettings.receivers ? nginx)
|
|
&& !(builtins.elem "nginx" leafSettings.service.pipelines.metrics.receivers);
|
|
}
|
|
{
|
|
# `metrics.address` is the spelling that looks right and is rejected by
|
|
# this collector version. The first clause is the control: without it a
|
|
# missing telemetry block would pass the port check vacuously.
|
|
name = "the hive collector binds its telemetry port through readers, not address";
|
|
ok =
|
|
let
|
|
m = hiveOtelSettings.service.telemetry.metrics;
|
|
in
|
|
!(m ? address) && (lib.head m.readers).pull.exporter.prometheus.port == 8888;
|
|
}
|
|
{
|
|
# A bind that loses the race with the outgoing process is the whole
|
|
# failure: nixpkgs ships `Restart = "always"` with no `RestartSec`, so
|
|
# without this the unit spends its five default attempts inside two
|
|
# seconds and lands in `start-limit-hit`, where it stops retrying. The
|
|
# third clause is the one that has to hold — `StartLimit*` are `[Unit]`
|
|
# settings that systemd ignores under `[Service]`, so a bound written
|
|
# into `serviceConfig` renders, deploys and does nothing. Asserted
|
|
# where nixpkgs puts it rather than where it was written, same as
|
|
# ./bao-grants.nix.
|
|
name = "the hive collector backs off a failed bind from [Unit], not [Service]";
|
|
ok =
|
|
let
|
|
u = hiveOtel.systemd.services.opentelemetry-collector;
|
|
in
|
|
u.serviceConfig.RestartSec or 0 > 0
|
|
&& toString u.unitConfig.StartLimitBurst == "12"
|
|
&& !(u.serviceConfig ? StartLimitBurst)
|
|
# The window has to outlast every attempt, or the burst is unreachable.
|
|
&& u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst;
|
|
}
|
|
{
|
|
# Collectors share a network namespace whenever they are co-located,
|
|
# and this port appears in no config the port-collision assertion can
|
|
# read — so equal defaults mean the second to start dies at `bind()`.
|
|
# Pinned as a case rather than an assertion: enforcing it belongs with
|
|
# the other port checks, not here.
|
|
#
|
|
# THREE tiers, because the store's container grew a collector of its
|
|
# own and this case could not see it: an UNDECLARED port is not an
|
|
# absent one, it is the binary's 8888, which is the hive tier's. That
|
|
# is why `portOf` falls back to 8888 rather than throwing — a tier that
|
|
# declares nothing has to fail this case loudly, not evaluate away.
|
|
name = "no two collector tiers claim the same self-telemetry port";
|
|
ok =
|
|
let
|
|
portOf =
|
|
s:
|
|
(lib.head (s.service.telemetry.metrics.readers or [ { } ])).pull.exporter.prometheus.port or 8888;
|
|
hivePort = portOf hiveOtelSettings;
|
|
swarmPort = portOf (otelSettings baoWithCollector);
|
|
baoPort = portOf (baoForwarderSettings baoWithHiveOtel);
|
|
in
|
|
hivePort != swarmPort && hivePort != baoPort && swarmPort != baoPort;
|
|
}
|
|
{
|
|
# The collector authenticates as the hive, so it reads its secret as
|
|
# the hive: the leaf `deploy.bao.clientCertFile` names, whose policy
|
|
# grants `swarm/hives/<hive>/*`. The last two clauses tie the unit's
|
|
# output to what the collector loads.
|
|
name = "the hive collector's secret is fetched from the store under the hive's own leaf";
|
|
ok =
|
|
let
|
|
u = hiveOtelFetch hiveOtelLeaf;
|
|
b = hiveOtelLeaf.services.hyperhive.deploy.bao;
|
|
file = hiveOtelLeaf.services.hyperhive.otel.clientSecretFile;
|
|
s = builtins.unsafeDiscardStringContext u.script;
|
|
in
|
|
hiveOtelLeaf.systemd.services ? swarm-bao-hive-otel-oidc
|
|
&& u.environment.BAO_CLIENT_CERT == b.clientCertFile
|
|
&& u.environment.BAO_CLIENT_KEY == b.clientKeyFile
|
|
&& lib.hasInfix "bao login -method=cert" s
|
|
&& lib.hasInfix "bao kv get -field=value secret/swarm/hives/h1/oidc/client" s
|
|
&& lib.hasInfix "atomic_write_secret 0400 \"\" ${file} " s
|
|
&& builtins.elem "swarm-client.secret:${file}" hiveOtelLeaf.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
|
|
&& builtins.elem "opentelemetry-collector.service" u.wantedBy
|
|
&& builtins.elem "opentelemetry-collector.service" u.before;
|
|
}
|
|
{
|
|
# `LoadCredential` copies the file at the collector's start, so a value
|
|
# landed later reaches it only through a restart. Retries every 30s
|
|
# (./lib/store-retry.nix) while the store is down or the secret
|
|
# unpublished.
|
|
name = "the hive collector's fetch retries and restarts the collector when the file is newer";
|
|
ok =
|
|
let
|
|
u = hiveOtelFetch hiveOtelLeaf;
|
|
s = builtins.unsafeDiscardStringContext u.script;
|
|
in
|
|
u.serviceConfig.Restart == "on-failure"
|
|
&& u.serviceConfig.RestartSec == 30
|
|
&& toString u.unitConfig.StartLimitBurst == "2880"
|
|
&& lib.hasInfix "refresh_unit opentelemetry-collector.service /var/lib/hive-otel-oidc/hive-h1.secret" s
|
|
&& lib.hasInfix "ActiveEnterTimestamp" s
|
|
&& lib.hasInfix ''if [ "$written_us" -le "$started_us" ]; then'' s;
|
|
}
|
|
{
|
|
# On a first boot the file does not exist until the fetch succeeds, so
|
|
# the collector fails `LoadCredential` into its start limit, i.e. the
|
|
# `failed` state. Only `reset-failed` clears that limit, so the fetch
|
|
# has to reset before it starts; `try-restart` acts on no failed unit.
|
|
# A never-active collector has an empty `ActiveEnterTimestamp`, read as
|
|
# 0, so the file always counts as newer. The collector only `wants` the
|
|
# fetch, so an unreachable store does not hold it back.
|
|
name = "the hive collector's fetch revives a collector that hit its start limit without the file";
|
|
ok =
|
|
let
|
|
s = builtins.unsafeDiscardStringContext (hiveOtelFetch hiveOtelLeaf).script;
|
|
c = hiveOtelLeaf.systemd.services.opentelemetry-collector;
|
|
failedArm = lib.last (lib.splitString ''is-failed --quiet "$unit"; then'' s);
|
|
in
|
|
lib.hasInfix ''is-failed --quiet "$unit"; then'' s
|
|
&& lib.hasInfix ''reset-failed "$unit"'' failedArm
|
|
&& lib.hasInfix ''start --no-block "$unit"'' (
|
|
lib.last (lib.splitString ''reset-failed "$unit"'' failedArm)
|
|
)
|
|
&& lib.hasInfix "started_us=0" s
|
|
&& !(builtins.elem "swarm-bao-hive-otel-oidc.service" (c.requires or [ ]));
|
|
}
|
|
{
|
|
# The store is the one route, on the IdP's own host too: no default into
|
|
# authelia's tree and no unit waiting for a file to appear there.
|
|
name = "the hive collector reads no secret out of authelia's tree and waits on no copied file";
|
|
ok =
|
|
let
|
|
file = hiveOtelAllLocal.services.hyperhive.otel.clientSecretFile;
|
|
c = hiveOtelAllLocal.systemd.services.opentelemetry-collector;
|
|
in
|
|
!(hiveOtelAllLocal.systemd.services ? hive-otel-client-secret)
|
|
&& !(hiveOtelLeaf.systemd.services ? hive-otel-client-secret)
|
|
&& !(lib.hasInfix "authelia" file)
|
|
&& hiveOtelAllLocal.systemd.services ? swarm-bao-hive-otel-oidc
|
|
&& !(builtins.elem "hive-otel-client-secret.service" (c.requires or [ ]));
|
|
}
|
|
{
|
|
# The writer and the reader spell the path independently; this is the
|
|
# one place that sees both rendered.
|
|
name = "the hive collector reads the path the secret publisher writes";
|
|
ok =
|
|
let
|
|
fetch = builtins.unsafeDiscardStringContext (hiveOtelFetch hiveOtelAllLocal).script;
|
|
publish = builtins.unsafeDiscardStringContext hiveOtelAllLocal.containers.swarm-authelia.config.systemd.services.swarm-secret-publish.script;
|
|
path = "secret/swarm/hives/h1/oidc/client";
|
|
in
|
|
lib.hasInfix "bao kv get -field=value ${path}" fetch && lib.hasInfix "bao kv put ${path} " publish;
|
|
}
|
|
{
|
|
# The store's units exist only on its host. The PKI unit is wanted, never
|
|
# required: a refused pki start fails a requirer's job with `dependency`.
|
|
name = "the hive collector's fetch waits for the store's container and PKI unit only where the store runs here";
|
|
ok =
|
|
let
|
|
storeUnits = m: [
|
|
"swarm-bao-pki.service"
|
|
"container@${m.services.hyperhive.swarm.bao.machine}.service"
|
|
];
|
|
here = hiveOtelFetch hiveOtelAllLocal;
|
|
elsewhere = hiveOtelFetch hiveOtelLeaf;
|
|
in
|
|
lib.all (n: lib.elem n here.after) (storeUnits hiveOtelAllLocal)
|
|
&& lib.elem "swarm-bao-pki.service" here.wants
|
|
&& !(lib.elem "swarm-bao-pki.service" here.requires)
|
|
&& lib.elem "container@${hiveOtelAllLocal.services.hyperhive.swarm.bao.machine}.service" here.wants
|
|
&& !(lib.any (n: lib.elem n (elsewhere.after ++ elsewhere.wants ++ elsewhere.requires)) (
|
|
storeUnits hiveOtelLeaf
|
|
));
|
|
}
|
|
{
|
|
# A secret delivered some other way is the operator's: no unit fetches
|
|
# into a file the collector does not load, or restarts it on one. Nothing
|
|
# restarts the collector when that file lands, so it waits for the file
|
|
# instead of failing `LoadCredential` into its start limit.
|
|
name = "a hand-named collector secret renders no store fetch, and the collector waits for the file";
|
|
ok =
|
|
let
|
|
waitsFor =
|
|
machine:
|
|
let
|
|
c = machine.systemd.services.opentelemetry-collector;
|
|
w = machine.systemd.services.hive-otel-client-secret;
|
|
in
|
|
builtins.elem "hive-otel-client-secret.service" c.requires
|
|
&& builtins.elem "hive-otel-client-secret.service" c.after
|
|
&& lib.hasInfix "/var/lib/hive-otel-oidc/client.secret" w.script;
|
|
in
|
|
!(hiveOtel.systemd.services ? swarm-bao-hive-otel-oidc)
|
|
&& !(baoWithHiveOtel.systemd.services ? swarm-bao-hive-otel-oidc)
|
|
&& builtins.elem "swarm-client.secret:/var/lib/hive-otel-oidc/client.secret" baoWithHiveOtel.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
|
|
&& waitsFor hiveOtel
|
|
&& waitsFor baoWithHiveOtel;
|
|
}
|
|
{
|
|
# ../host-modules/glue-hive-otel-authelia.nix orders the wait after
|
|
# authelia's container. `hiveOtel` has the IdP; `baoWithHiveOtel` waits
|
|
# on the same kind of file without it, and must not name a container
|
|
# that is not there — `Requires=` on an absent unit fails the job.
|
|
name = "the hive collector's secret wait is ordered after authelia's container only where authelia runs here";
|
|
ok =
|
|
let
|
|
container = "container@${hiveOtel.services.hyperhive.swarm.authelia.machine}.service";
|
|
wait = machine: machine.systemd.services.hive-otel-client-secret;
|
|
in
|
|
builtins.elem container (wait hiveOtel).after
|
|
&& builtins.elem container (wait hiveOtel).requires
|
|
&& !(builtins.elem container (wait baoWithHiveOtel).after)
|
|
&& !(builtins.elem container (wait baoWithHiveOtel).requires);
|
|
}
|
|
];
|
|
in
|
|
runGroup "hive-otel" cases
|