vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-bao: keep the store forwarder's OIDC secret on the host's /run, not in the container's root

Browse source 
swarm-bao-forwarder-oidc still reads the secret from bao under the
forwarder's own leaf, but now writes it to /run/swarm-bao-forwarder-oidc on
the host (tmpfs), which the swarm-bao container bind-mounts read-only for the
collector's LoadCredential. Before, it wrote it to
/var/lib/nixos-containers/swarm-bao/var/lib/swarm-bao-otel-oidc/<id>.secret on
persistent disk, where a superseded secret stayed after rotation and went
into backups of the container root. Host tmpfiles now removes that old file.

Rotation is unchanged: the ten-minute timer rewrites the file when the value
differs and refresh_consumer restarts the forwarder. After a reboot the
secret is fetched again; the file is not pre-created, so until the store
answers the forwarder stays down (LoadCredential refuses a missing source) and
resumes from its journal cursor afterwards. The container now starts after
systemd-tmpfiles-resetup, since the new bind source is made there.

Part of #4998 (slice 3: forwarder).
This commit is contained in:
atlas 2026-10-15 22:36:28 +02:00
commit 6f5bd1dc5d
3 changed files with 80 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

78
nix/host-modules/swarm-bao.nix
View file

@ -1401,12 +1401,17 @@ let
# `swarm-forwarder-oidc` role's grant is this one path, so any other answers
# 403 however correct it looks.
forwarderStoreSecretPath = "secret/swarm/services/${cfg.otel.clientId}/oidc/client";
# At rest in the container's tree, written by the host unit below — under
# /var/lib rather than /run, because a secret that evaporates on reboot
# turns a working export into an intermittent one.
forwarderSecretInContainer = "/var/lib/swarm-bao-otel-oidc/${cfg.otel.clientId}.secret";
forwarderHostSecretPath = "/var/lib/nixos-containers/${cfg.machine}${forwarderSecretInContainer}";
forwarderHostSecretDir = builtins.dirOf forwarderHostSecretPath;
# On the host's /run (tmpfs), so the secret never reaches persistent disk:
# after a reboot `swarm-bao-forwarder-oidc` fetches it again. The container
# sees the directory read-only at the same path. The file is NOT created
# ahead of that fetch: `LoadCredential` refuses a missing source, so until
# the store answers the forwarder stays down and resumes from its saved
# read cursor. A placeholder file would start it, and its refused pushes
# would be dropped once the exporter's retry window ends.
forwarderSecretDir = "/run/swarm-bao-forwarder-oidc";
forwarderSecret = "${forwarderSecretDir}/${cfg.otel.clientId}.secret";
# Where the secret sat on persistent disk before it moved to /run.
forwarderLegacyHostSecretPath = "/var/lib/nixos-containers/${cfg.machine}/var/lib/swarm-bao-otel-oidc/${cfg.otel.clientId}.secret";
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
refreshConsumer = import ./lib/refresh-consumer.nix { };
@ -2738,8 +2743,9 @@ in
# THE delivery unit for the forwarder's own OIDC client secret, copied
# in shape from ./swarm-otel.nix's `swarm-bao-otel-oidc`: a cert login
# that fails LOUDLY, since every state it fails on is one a retry fixes,
# then a read, then a root-owned file in the container's tree that
# `LoadCredential` re-exposes to the collector's dynamic user.
# then a read, then a root-owned file on the host's /run that the
# container bind-mounts read-only and `LoadCredential` re-exposes to the
# collector's dynamic user.
#
# ⚠️ ORDERED AFTER the store's container and deliberately NOT before it,
# which is the one line that differs from the unit it copies. That one
@ -2748,12 +2754,11 @@ in
# process that cannot start until this finishes. There is no bootstrap
# cycle behind it — the identity used here is the static
# `forwarder-oidc.pem` leaf from `swarm-bao-pki`, not anything the store
# issues — only an ordering one, and the cost is bounded: the file is
# under /var/lib, so it survives reboots and only a FIRST boot has the
# collector starting before it exists. `LoadCredential` refuses to start
# a unit whose source is missing, so that boot is a collector that
# restarts, says so each time, and comes up the moment this lands. Loud
# and self-healing rather than silently exporting without a credential.
# issues — only an ordering one. The file is on tmpfs, so EVERY boot has
# the collector starting before it exists: `LoadCredential` refuses to
# start a unit whose source is missing, so the collector restarts, says
# so each time, and comes up the moment this lands. Loud and
# self-healing rather than silently exporting without a credential.
systemd.services.swarm-bao-forwarder-oidc = {
description = "fetch the secret store forwarder's OIDC client secret from the store";
after = [
@ -2837,18 +2842,25 @@ in
# a program: `printf` is bash's own, so the plaintext never becomes
# an argument in /proc the way `install <<<"$secret"` or an `echo`
# from `path` would.
install -d -m 0755 ${lib.escapeShellArg forwarderHostSecretDir}
if secret_differs ${lib.escapeShellArg forwarderHostSecretPath} "$secret"; then
atomic_write_secret 0400 root:root ${lib.escapeShellArg forwarderHostSecretPath} "$secret"
install -d -m 0700 ${lib.escapeShellArg forwarderSecretDir}
if secret_differs ${lib.escapeShellArg forwarderSecret} "$secret"; then
atomic_write_secret 0400 root:root ${lib.escapeShellArg forwarderSecret} "$secret"
fi
# `LoadCredential` copies the file at start only: a rotated secret
# reaches the forwarder by restarting it.
refresh_consumer ${lib.escapeShellArg cfg.machine} opentelemetry-collector.service ${lib.escapeShellArg forwarderHostSecretPath}
refresh_consumer ${lib.escapeShellArg cfg.machine} opentelemetry-collector.service ${lib.escapeShellArg forwarderSecret}
'';
};
systemd.timers.swarm-bao-forwarder-oidc = storeRefreshTimer "re-read the secret store forwarder's OIDC client secret from the store";
systemd.tmpfiles.rules = [
# The container's bind source, so it exists when nspawn sets the mount
# up. Not the file: see `forwarderSecretDir`.
"d ${forwarderSecretDir} 0700 root root -"
"r ${forwarderLegacyHostSecretPath}"
];
})
(lib.mkIf (deployCfg.bao.enable && haveServerTls) {
@ -3721,7 +3733,15 @@ in
# The CA bind source is written at runtime by a host unit, so the
# container has to start after it — otherwise nspawn sets up a mount
# over a file that does not exist yet.
systemd.services."container@${cfg.machine}" = caTrust.containerOrdering;
#
# The forwarder's secret directory is a bind source too, made by
# tmpfiles; on a switch the container restarts in the same transaction
# as `systemd-tmpfiles-resetup`, so it starts after it. A container that
# cannot start is a store that is down.
systemd.services."container@${cfg.machine}" = lib.mkMerge [
caTrust.containerOrdering
{ after = [ "systemd-tmpfiles-resetup.service" ]; }
];
containers.${cfg.machine} = {
autoStart = true;
@ -3734,6 +3754,13 @@ in
# /var, systemd owns `${stateDir}` through `StateDirectory=`, and
# binding over it is what breaks the unit.
bindMounts = {
# The forwarder's OIDC secret, written by `swarm-bao-forwarder-oidc`
# on the host's tmpfs; read-only because the container only loads it.
${forwarderSecretDir} = {
hostPath = forwarderSecretDir;
isReadOnly = true;
};
# Read-only: host units write it (`swarm-bao-certs`, and
# `swarm-bao-agent-pki` for the agent CA and the listener bundle), and
# the store has no reason to modify its own identity.
@ -4230,17 +4257,18 @@ in
# How the secret crosses the sandbox. The host unit
# `swarm-bao-forwarder-oidc` leaves a root-owned 0400 file at this
# path in the container's tree; systemd reads it as root before
# the sandbox exists and re-exposes it to whichever uid
# `DynamicUser` picked, under the directory the extension above
# names through `${env:CREDENTIALS_DIRECTORY}`.
# path on the host's /run, which the container bind-mounts
# read-only; systemd reads it as root before the sandbox exists
# and re-exposes it to whichever uid `DynamicUser` picked, under
# the directory the extension above names through
# `${env:CREDENTIALS_DIRECTORY}`.
#
# ⚠️ Unconditional, and a missing source is a unit that refuses to
# start. That is the intended shape rather than an oversight: this
# forwarder has one way to deliver, so a boot where the secret has
# not arrived yet is a collector that says so and retries, not one
# that quietly exports into a 401. The restart is what closes the
# first-boot window the host unit's own comment describes.
# post-boot window the host unit's own comment describes.
#
# ⚠️ `Restart` is deliberately NOT set here, same as the sibling
# collector in ./swarm-otel.nix: nixpkgs' own
@ -4254,7 +4282,7 @@ in
# is still delivering the secret, leaving it failed rather than
# retrying.
systemd.services.opentelemetry-collector.serviceConfig = {
LoadCredential = [ "${forwarderCredentialId}:${forwarderSecretInContainer}" ];
LoadCredential = [ "${forwarderCredentialId}:${forwarderSecret}" ];
RestartSec = 15;
};
};

25
nix/module-eval/bao-otel-collector.nix
View file

@ -368,9 +368,32 @@ let
e.client_secret_file == "\${env:CREDENTIALS_DIRECTORY}/oidc-client-secret"
&&
unit.LoadCredential == [
"oidc-client-secret:/var/lib/swarm-bao-otel-oidc/swarm-bao-collector.secret"
"oidc-client-secret:/run/swarm-bao-forwarder-oidc/swarm-bao-collector.secret"
];
}
{
# A copy on persistent disk outlives the secret's rotation and lands in
# backups of the container's root. The file is deliberately not created
# ahead of the fetch: a placeholder would start the forwarder and have
# its pushes refused instead of leaving it down until the store answers.
name = "the forwarder's client secret is written to the host's /run only, and the old copy removed";
ok =
let
m = baoWithCollector;
script = m.systemd.services.swarm-bao-forwarder-oidc.script;
mount = m.containers.swarm-bao.bindMounts."/run/swarm-bao-forwarder-oidc";
rules = m.systemd.tmpfiles.rules;
in
lib.hasInfix "atomic_write_secret 0400 root:root /run/swarm-bao-forwarder-oidc/swarm-bao-collector.secret" script
&& !(lib.hasInfix "/var/lib/nixos-containers" script)
&& !(lib.hasInfix "/var/lib/swarm-bao-otel-oidc" script)
&& mount.hostPath == "/run/swarm-bao-forwarder-oidc"
&& mount.isReadOnly
&& lib.elem "d /run/swarm-bao-forwarder-oidc 0700 root root -" rules
&& lib.elem "r /var/lib/nixos-containers/swarm-bao/var/lib/swarm-bao-otel-oidc/swarm-bao-collector.secret" rules
&& !(lib.any (r: lib.hasPrefix "f /run/swarm-bao-forwarder-oidc" r) rules)
&& lib.elem "systemd-tmpfiles-resetup.service" m.systemd.services."container@swarm-bao".after;
}
{
# Same 403-not-a-miss property the grafana and matrix readers are pinned
# for: the reader's grant is one path under `services`, so a secret