vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(swarm-otel): deliver the collector's client secret from authelia

Browse source 
The scrape config names a client_secret_file; this is what puts a file
there. A host oneshot copies authelia's minted secret between the two
container trees — a copy and not a bind mount, because the secret does
not exist until authelia's first boot and nixos-container refuses to
start on a missing bind source, which on a fresh swarm is a permanent
stall presenting as broken metrics.

The collector runs under DynamicUser and the prometheus receiver opens
client_secret_file itself, as that user, so there is no uid to hand the
file to. LoadCredential reads it as root before the sandbox exists and
re-exposes it under a path that does not depend on which uid the unit
got; the scrape config points there. Both spellings derive from one
binding, since a mismatch is a file the collector cannot open and
nothing but a runtime 401 would say so.
This commit is contained in:
atlas 2026-08-24 13:46:48 +02:00
commit 2e06957b32
1 changed files with 95 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

104
nix/host-modules/swarm-otel.nix
View file

@ -70,11 +70,30 @@ let
# error rather than a warning nobody reads.
parsePublished = url: builtins.match "https://([^/]+)(/.*)" url;
# Where the collector reads its own client secret. Under /var/lib and not
# /run for the same reason the forge and matrix secrets are: the collector
# may start before the delivery unit on a later boot, and a secret that
# evaporates on reboot turns a working scrape into an intermittent one.
collectorSecretPath = "/var/lib/swarm-otel/${cfg.clientId}.secret";
# The client secret takes three names, and the reason is `DynamicUser`.
#
# Upstream's collector unit runs with `DynamicUser = true`, and the
# prometheus receiver opens `client_secret_file` ITSELF, at runtime, as that
# user — so there is no stable uid to hand a file to, and the root-owned
# 0400 shape `hive-matrix-oidc-secret` delivers to would be unreadable.
# (Matrix gets away with it because `LoadCredential` reads the file as root
# before the sandbox exists, and tuwunel never opens that path itself.)
#
# `LoadCredential` solves both halves: systemd reads the file as root and
# re-exposes it to the dynamic user under a path that does not depend on
# which uid it turned out to be.
#
# At rest in the container's tree — written by the host oneshot below.
# Under /var/lib and not /run because the collector may start before the
# delivery unit on a later boot, and a secret that evaporates on reboot
# turns a working scrape into an intermittent one.
collectorSecretInContainer = "/var/lib/swarm-otel-oidc/${cfg.clientId}.secret";
collectorCredentialId = "oidc-client-secret";
# What the scrape config points at. ⚠️ This path and the `LoadCredential`
# id below are one fact spelled twice by systemd's design — both derive from
# `collectorCredentialId` so they cannot drift; a mismatch is a file the
# collector cannot open, discovered at runtime and nowhere else.
collectorSecretPath = "/run/credentials/opentelemetry-collector.service/${collectorCredentialId}";
# `attrNames` is sorted, so this is a function of the hive SET and not of
# the order anyone wrote it in.
@ -378,6 +397,63 @@ in
}
];
# Deliver the collector's client secret from authelia's container into
# this one. On the HOST because that is the only place both container
# trees are addressable: they share this host's network namespace, which
# makes them feel co-located, but their filesystem roots are separate.
#
# ⚠️ Deliberately a copy and not a `bindMounts` entry. nixos-container
# refuses to start when a bind source is missing, and this secret does not
# exist until authelia's first boot has minted it — so binding it would
# make the collector wait on a file that waits on a container that starts
# after it. On a fresh swarm that is a permanent stall presenting as
# "metrics are broken", several layers from its cause.
systemd.services.swarm-otel-oidc-secret =
lib.mkIf (autheliaCfg.enable && cfg.publishedScrapeTargets != { })
{
description = "deliver the swarm collector's OIDC client secret from authelia";
after = [ "container@${autheliaCfg.machine}.service" ];
requires = [ "container@${autheliaCfg.machine}.service" ];
before = [ "container@${cfg.machine}.service" ];
wantedBy = [ "container@${cfg.machine}.service" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
SyslogIdentifier = "swarm-otel-oidc-secret";
# Longer than the bounded wait below, and that is the point:
# `DefaultTimeoutStartSec` is 90s, so without this systemd kills
# the unit before it can emit the message naming the file it was
# waiting for — the failure then reads as a timeout with no cause.
TimeoutStartSec = "180s";
};
path = [ pkgs.coreutils ];
script = ''
set -euo pipefail
src=${lib.escapeShellArg "${autheliaCfg.hostClientSecretDir}/${cfg.clientId}.secret"}
dst=${lib.escapeShellArg "/var/lib/nixos-containers/${cfg.machine}${collectorSecretInContainer}"}
# authelia's container is up, but its first-boot generator may
# still be minting. Bounded wait, then fail: skipping silently
# produces a collector whose scrape gets a 401 forever, which is
# the failure this whole design exists to make impossible.
for _ in $(seq 1 60); do
[ -s "$src" ] && break
sleep 2
done
if [ ! -s "$src" ]; then
echo "authelia has not minted $src after 120s" >&2
exit 1
fi
# root-owned 0400. The collector runs under `DynamicUser`, so
# there is no uid to give it to — `LoadCredential` reads this as
# root before the sandbox exists and re-exposes it to whichever
# uid the unit got.
install -D -m 0400 -o root -g root "$src" "$dst"
'';
};
# The CA bind source is written at runtime by a host unit, so the
# container has to start after it — otherwise nspawn sets up a mount
# over a file that does not exist yet.
@ -863,10 +939,20 @@ in
# environment variable without being read by nix, written to the
# store, or passed in argv.
systemd.services.opentelemetry-collector.serviceConfig =
lib.optionalAttrs (otelCfg.headersCredential != null)
{
EnvironmentFile = otelCfg.headersCredential;
};
lib.optionalAttrs (otelCfg.headersCredential != null) {
EnvironmentFile = otelCfg.headersCredential;
}
# The other credential, and the other direction: the one above
# authenticates this collector's export onward, this one
# authenticates it to a service it scrapes.
#
# ⚠️ Only where a published target exists. `LoadCredential` on a
# missing source is a unit that refuses to start, so declaring it
# unconditionally would take the collector down on every hive that
# scrapes nothing published — the empty case is the shipped one.
// lib.optionalAttrs (cfg.publishedScrapeTargets != { }) {
LoadCredential = [ "${collectorCredentialId}:${collectorSecretInContainer}" ];
};
};
};
};