vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-otel: keep the collector's OIDC secret on the host's /run, not in the container's root

Browse source 
swarm-bao-otel-oidc still reads the secret from bao under the collector's
own leaf, but now writes it to /run/swarm-otel-oidc on the host (tmpfs),
which the container bind-mounts read-only for the collector's
LoadCredential. Before, it wrote it to
/var/lib/nixos-containers/swarm-otel/var/lib/swarm-otel-oidc/<id>.secret on
persistent disk, where a superseded secret stayed after rotation and went
into backups of the container root. Host tmpfiles now removes that old file.

Rotation is unchanged: the ten-minute timer rewrites the file when the value
differs and refresh_consumer restarts the collector. After a reboot the
secret is fetched again; until the store answers the collector starts on an
empty file and the stores refuse its pushes and scrapes.

Part of #4998 (slice 2a).
This commit is contained in:
atlas 2026-10-15 20:43:10 +02:00 • committed by mara
commit 7539ef82f7
3 changed files with 71 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

60
nix/host-modules/swarm-otel.nix
View file

@ -140,18 +140,21 @@ let
# re-exposes it to the dynamic user under a path that does not depend on
# which uid it turned out to be.
#
# At rest in the container's tree — written by the host oneshot below.
# Under /var/lib and not /run because the collector may start before the
# delivery unit on a later boot, and a secret that evaporates on reboot
# turns a working scrape into an intermittent one.
collectorSecretInContainer = "/var/lib/swarm-otel-oidc/${cfg.clientId}.secret";
# The same file seen from the host, which is where the delivery unit
# writes it — spelled once for the same reason ./swarm-grafana.nix spells
# its own pair once: the unit that writes it and the option that names it
# are a few hundred lines apart, and a collector reading a path nothing
# On the host's /run (tmpfs), so the secret never reaches persistent disk:
# after a reboot the delivery unit below fetches it again. The container
# sees the directory read-only at the same path. The file starts out empty
# (tmpfiles, below) so `LoadCredential` has a source before the first
# fetch; a collector that starts on it is refused by the stores until the
# fetched secret restarts it.
#
# Spelled once, for the unit that writes it and the option that names it,
# which are a few hundred lines apart: a collector reading a path nothing
# writes is an export that fails with nothing in any log about the file.
collectorHostSecretPath = "/var/lib/nixos-containers/${cfg.machine}${collectorSecretInContainer}";
collectorHostSecretDir = builtins.dirOf collectorHostSecretPath;
collectorSecretDir = "/run/swarm-otel-oidc";
collectorSecretInContainer = "${collectorSecretDir}/${cfg.clientId}.secret";
# Where the secret sat on persistent disk before it moved to /run.
collectorLegacySecretInContainer = "/var/lib/swarm-otel-oidc/${cfg.clientId}.secret";
collectorLegacyHostSecretPath = "/var/lib/nixos-containers/${cfg.machine}${collectorLegacySecretInContainer}";
collectorCredentialId = "oidc-client-secret";
# What the scrape config points at. ⚠️ This path and the `LoadCredential`
# id below are one fact spelled twice by systemd's design — both derive from
@ -472,6 +475,11 @@ in
${atomicWriteSecret}
${refreshConsumer}
# The container's bind source, so it starts even when the store does
# not answer; tmpfiles creates it too, but unordered with the
# container on a switch.
install -d -m 0700 ${lib.escapeShellArg collectorSecretDir}
# `bao`'s own message is the only thing separating a missing value
# from a refused identity from an unreachable host. This unit's
# degraded mode is correct for all three, so it reports which one
@ -516,14 +524,13 @@ in
# `path` would. The collector runs under `DynamicUser`, so there is
# no uid to give this to — `LoadCredential` reads it as root before
# the sandbox exists and re-exposes it to whichever uid the unit got.
install -d -m 0755 ${lib.escapeShellArg collectorHostSecretDir}
if secret_differs ${lib.escapeShellArg collectorHostSecretPath} "$secret"; then
atomic_write_secret 0400 root:root ${lib.escapeShellArg collectorHostSecretPath} "$secret"
if secret_differs ${lib.escapeShellArg collectorSecretInContainer} "$secret"; then
atomic_write_secret 0400 root:root ${lib.escapeShellArg collectorSecretInContainer} "$secret"
fi
# `LoadCredential` copies the file at start only, and a collector that
# started without it is in its start limit.
refresh_consumer ${lib.escapeShellArg cfg.machine} opentelemetry-collector.service ${lib.escapeShellArg collectorHostSecretPath}
refresh_consumer ${lib.escapeShellArg cfg.machine} opentelemetry-collector.service ${lib.escapeShellArg collectorSecretInContainer}
'';
};
@ -540,6 +547,20 @@ in
lib.mkDefault collectorSecretInContainer
);
systemd.tmpfiles.rules = lib.optionals haveClientIdentity (
[
# The bind source must exist when the container starts.
"d ${collectorSecretDir} 0700 root root -"
# `LoadCredential=` fails the unit when the file is missing; an empty
# one lets the collector start and be refused by the stores.
"f ${collectorSecretInContainer} 0400 root root -"
]
# Unless a deployment delivers its own secret to that very path.
++ lib.optional (
deployCfg.swarm-otel.clientSecretFile != collectorLegacySecretInContainer
) "r ${collectorLegacyHostSecretPath}"
);
# The CA bind source is written at runtime by a host unit, so the
# container has to start after it — otherwise nspawn sets up a mount
# over a file that does not exist yet.
@ -788,7 +809,14 @@ in
# does not exist makes nixos-container refuse to start the container,
# which is a stall several layers from its cause.
bindMounts =
lib.optionalAttrs (otelCfg.headersCredential != null) {
# The collector's OIDC secret, where the delivery unit writes it.
lib.optionalAttrs haveClientIdentity {
${collectorSecretDir} = {
hostPath = collectorSecretDir;
isReadOnly = true;
};
}
// lib.optionalAttrs (otelCfg.headersCredential != null) {
${otelCfg.headersCredential} = {
hostPath = otelCfg.headersCredential;
isReadOnly = true;

26
nix/module-eval/swarm-otel-core.nix
View file

@ -105,6 +105,32 @@ let
lib.hasInfix "secret/swarm/services/swarm-collector/oidc/client" s
&& !(lib.hasInfix "secret/swarm/hives/" s);
}
{
# A copy on persistent disk outlives the secret's rotation and lands in
# backups of the container's root.
name = "the collector's OIDC secret is written to the host's /run only, and the old copy removed";
ok =
let
m = otelBaoRemoteAuthelia;
script = m.systemd.services.swarm-bao-otel-oidc.script;
mount = m.containers.swarm-otel.bindMounts."/run/swarm-otel-oidc";
rules = m.systemd.tmpfiles.rules;
in
lib.hasInfix "atomic_write_secret 0400 root:root /run/swarm-otel-oidc/swarm-collector.secret" script
&& !(lib.hasInfix "/var/lib/nixos-containers" script)
&& !(lib.hasInfix "/var/lib/swarm-otel-oidc" script)
&& mount.hostPath == "/run/swarm-otel-oidc"
&& mount.isReadOnly
&&
m.services.hyperhive.deploy.swarm-otel.clientSecretFile
== "/run/swarm-otel-oidc/swarm-collector.secret"
&& lib.elem "d /run/swarm-otel-oidc 0700 root root -" rules
&& lib.elem "f /run/swarm-otel-oidc/swarm-collector.secret 0400 root root -" rules
&& lib.elem "r /var/lib/nixos-containers/swarm-otel/var/lib/swarm-otel-oidc/swarm-collector.secret" rules
&&
lib.any (c: c == "oidc-client-secret:/run/swarm-otel-oidc/swarm-collector.secret")
m.containers.swarm-otel.config.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential;
}
{
# These exporters must not be gated on the stores' PER-HOST enables: a
# collector that did not share a host with them would render none at all