swarm-bao: keep the store forwarder's OIDC secret on the host's /run, not in the container's root
swarm-bao-forwarder-oidc still reads the secret from bao under the forwarder's own leaf, but now writes it to /run/swarm-bao-forwarder-oidc on the host (tmpfs), which the swarm-bao container bind-mounts read-only for the collector's LoadCredential. Before, it wrote it to /var/lib/nixos-containers/swarm-bao/var/lib/swarm-bao-otel-oidc/<id>.secret on persistent disk, where a superseded secret stayed after rotation and went into backups of the container root. Host tmpfiles now removes that old file. Rotation is unchanged: the ten-minute timer rewrites the file when the value differs and refresh_consumer restarts the forwarder. After a reboot the secret is fetched again; the file is not pre-created, so until the store answers the forwarder stays down (LoadCredential refuses a missing source) and resumes from its journal cursor afterwards. The container now starts after systemd-tmpfiles-resetup, since the new bind source is made there. Part of #4998 (slice 3: forwarder).
This commit is contained in:
atlas
parent
a1f99c77f8
commit
6f5bd1dc5d
3 changed files with 80 additions and 29 deletions
|
|
@ -112,7 +112,7 @@ The **secret store's own** collector — the forwarder inside the `swarm-bao`
|
|||
container — needs a delivery step too, and it takes the same route with one
|
||||
principal of its own: `swarm-bao-forwarder-oidc.service` reads
|
||||
`swarm/services/<client-id>/oidc/client` under a leaf of its own and
|
||||
lands it in the container's tree, where `LoadCredential` hands it to the
|
||||
writes it to `/run/swarm-bao-forwarder-oidc` on the host's tmpfs, which the container bind-mounts read-only and where `LoadCredential` hands it to the
|
||||
collector. The client id is its own
|
||||
(`services.hyperhive.swarm.bao.otel.clientId`), registered by
|
||||
`glue-swarm-bao-otel-oidc-client.nix` wherever authelia runs, and the swarm
|
||||
|
|
@ -123,11 +123,11 @@ this forwarder borrows neither the swarm collector's client nor
|
|||
⚠️ **This one has no degraded mode, and the difference is structural.** The
|
||||
route it exports over accepts an authenticated request and nothing else, so a
|
||||
forwarder without the secret has nothing to fall back to: the unit fails and
|
||||
retries, and the collector refuses to start until the file arrives rather than
|
||||
retries, and the collector refuses to start until the file arrives, after every boot, rather than
|
||||
exporting into a 401 while reporting healthy. Its ordering differs from the
|
||||
unit above it for the same structural reason — it reads the store inside the
|
||||
container it delivers into, so it runs **after** that container rather than
|
||||
before it, and only a first boot ever waits.
|
||||
before it. The fetch creates the file, so when the store stays sealed or unreachable after a boot the forwarder stays down, then resumes from its saved journal cursor once the store answers.
|
||||
|
||||
### The queue's callout nkeys
|
||||
|
||||
|
|
|
|||
|
|
@ -1401,12 +1401,17 @@ let
|
|||
# `swarm-forwarder-oidc` role's grant is this one path, so any other answers
|
||||
# 403 however correct it looks.
|
||||
forwarderStoreSecretPath = "secret/swarm/services/${cfg.otel.clientId}/oidc/client";
|
||||
# At rest in the container's tree, written by the host unit below — under
|
||||
# /var/lib rather than /run, because a secret that evaporates on reboot
|
||||
# turns a working export into an intermittent one.
|
||||
forwarderSecretInContainer = "/var/lib/swarm-bao-otel-oidc/${cfg.otel.clientId}.secret";
|
||||
forwarderHostSecretPath = "/var/lib/nixos-containers/${cfg.machine}${forwarderSecretInContainer}";
|
||||
forwarderHostSecretDir = builtins.dirOf forwarderHostSecretPath;
|
||||
# On the host's /run (tmpfs), so the secret never reaches persistent disk:
|
||||
# after a reboot `swarm-bao-forwarder-oidc` fetches it again. The container
|
||||
# sees the directory read-only at the same path. The file is NOT created
|
||||
# ahead of that fetch: `LoadCredential` refuses a missing source, so until
|
||||
# the store answers the forwarder stays down and resumes from its saved
|
||||
# read cursor. A placeholder file would start it, and its refused pushes
|
||||
# would be dropped once the exporter's retry window ends.
|
||||
forwarderSecretDir = "/run/swarm-bao-forwarder-oidc";
|
||||
forwarderSecret = "${forwarderSecretDir}/${cfg.otel.clientId}.secret";
|
||||
# Where the secret sat on persistent disk before it moved to /run.
|
||||
forwarderLegacyHostSecretPath = "/var/lib/nixos-containers/${cfg.machine}/var/lib/swarm-bao-otel-oidc/${cfg.otel.clientId}.secret";
|
||||
|
||||
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
|
||||
refreshConsumer = import ./lib/refresh-consumer.nix { };
|
||||
|
|
@ -2738,8 +2743,9 @@ in
|
|||
# THE delivery unit for the forwarder's own OIDC client secret, copied
|
||||
# in shape from ./swarm-otel.nix's `swarm-bao-otel-oidc`: a cert login
|
||||
# that fails LOUDLY, since every state it fails on is one a retry fixes,
|
||||
# then a read, then a root-owned file in the container's tree that
|
||||
# `LoadCredential` re-exposes to the collector's dynamic user.
|
||||
# then a read, then a root-owned file on the host's /run that the
|
||||
# container bind-mounts read-only and `LoadCredential` re-exposes to the
|
||||
# collector's dynamic user.
|
||||
#
|
||||
# ⚠️ ORDERED AFTER the store's container and deliberately NOT before it,
|
||||
# which is the one line that differs from the unit it copies. That one
|
||||
|
|
@ -2748,12 +2754,11 @@ in
|
|||
# process that cannot start until this finishes. There is no bootstrap
|
||||
# cycle behind it — the identity used here is the static
|
||||
# `forwarder-oidc.pem` leaf from `swarm-bao-pki`, not anything the store
|
||||
# issues — only an ordering one, and the cost is bounded: the file is
|
||||
# under /var/lib, so it survives reboots and only a FIRST boot has the
|
||||
# collector starting before it exists. `LoadCredential` refuses to start
|
||||
# a unit whose source is missing, so that boot is a collector that
|
||||
# restarts, says so each time, and comes up the moment this lands. Loud
|
||||
# and self-healing rather than silently exporting without a credential.
|
||||
# issues — only an ordering one. The file is on tmpfs, so EVERY boot has
|
||||
# the collector starting before it exists: `LoadCredential` refuses to
|
||||
# start a unit whose source is missing, so the collector restarts, says
|
||||
# so each time, and comes up the moment this lands. Loud and
|
||||
# self-healing rather than silently exporting without a credential.
|
||||
systemd.services.swarm-bao-forwarder-oidc = {
|
||||
description = "fetch the secret store forwarder's OIDC client secret from the store";
|
||||
after = [
|
||||
|
|
@ -2837,18 +2842,25 @@ in
|
|||
# a program: `printf` is bash's own, so the plaintext never becomes
|
||||
# an argument in /proc the way `install <<<"$secret"` or an `echo`
|
||||
# from `path` would.
|
||||
install -d -m 0755 ${lib.escapeShellArg forwarderHostSecretDir}
|
||||
if secret_differs ${lib.escapeShellArg forwarderHostSecretPath} "$secret"; then
|
||||
atomic_write_secret 0400 root:root ${lib.escapeShellArg forwarderHostSecretPath} "$secret"
|
||||
install -d -m 0700 ${lib.escapeShellArg forwarderSecretDir}
|
||||
if secret_differs ${lib.escapeShellArg forwarderSecret} "$secret"; then
|
||||
atomic_write_secret 0400 root:root ${lib.escapeShellArg forwarderSecret} "$secret"
|
||||
fi
|
||||
|
||||
# `LoadCredential` copies the file at start only: a rotated secret
|
||||
# reaches the forwarder by restarting it.
|
||||
refresh_consumer ${lib.escapeShellArg cfg.machine} opentelemetry-collector.service ${lib.escapeShellArg forwarderHostSecretPath}
|
||||
refresh_consumer ${lib.escapeShellArg cfg.machine} opentelemetry-collector.service ${lib.escapeShellArg forwarderSecret}
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.timers.swarm-bao-forwarder-oidc = storeRefreshTimer "re-read the secret store forwarder's OIDC client secret from the store";
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
# The container's bind source, so it exists when nspawn sets the mount
|
||||
# up. Not the file: see `forwarderSecretDir`.
|
||||
"d ${forwarderSecretDir} 0700 root root -"
|
||||
"r ${forwarderLegacyHostSecretPath}"
|
||||
];
|
||||
})
|
||||
|
||||
(lib.mkIf (deployCfg.bao.enable && haveServerTls) {
|
||||
|
|
@ -3721,7 +3733,15 @@ in
|
|||
# The CA bind source is written at runtime by a host unit, so the
|
||||
# container has to start after it — otherwise nspawn sets up a mount
|
||||
# over a file that does not exist yet.
|
||||
systemd.services."container@${cfg.machine}" = caTrust.containerOrdering;
|
||||
#
|
||||
# The forwarder's secret directory is a bind source too, made by
|
||||
# tmpfiles; on a switch the container restarts in the same transaction
|
||||
# as `systemd-tmpfiles-resetup`, so it starts after it. A container that
|
||||
# cannot start is a store that is down.
|
||||
systemd.services."container@${cfg.machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
{ after = [ "systemd-tmpfiles-resetup.service" ]; }
|
||||
];
|
||||
|
||||
containers.${cfg.machine} = {
|
||||
autoStart = true;
|
||||
|
|
@ -3734,6 +3754,13 @@ in
|
|||
# /var, systemd owns `${stateDir}` through `StateDirectory=`, and
|
||||
# binding over it is what breaks the unit.
|
||||
bindMounts = {
|
||||
# The forwarder's OIDC secret, written by `swarm-bao-forwarder-oidc`
|
||||
# on the host's tmpfs; read-only because the container only loads it.
|
||||
${forwarderSecretDir} = {
|
||||
hostPath = forwarderSecretDir;
|
||||
isReadOnly = true;
|
||||
};
|
||||
|
||||
# Read-only: host units write it (`swarm-bao-certs`, and
|
||||
# `swarm-bao-agent-pki` for the agent CA and the listener bundle), and
|
||||
# the store has no reason to modify its own identity.
|
||||
|
|
@ -4230,17 +4257,18 @@ in
|
|||
|
||||
# How the secret crosses the sandbox. The host unit
|
||||
# `swarm-bao-forwarder-oidc` leaves a root-owned 0400 file at this
|
||||
# path in the container's tree; systemd reads it as root before
|
||||
# the sandbox exists and re-exposes it to whichever uid
|
||||
# `DynamicUser` picked, under the directory the extension above
|
||||
# names through `${env:CREDENTIALS_DIRECTORY}`.
|
||||
# path on the host's /run, which the container bind-mounts
|
||||
# read-only; systemd reads it as root before the sandbox exists
|
||||
# and re-exposes it to whichever uid `DynamicUser` picked, under
|
||||
# the directory the extension above names through
|
||||
# `${env:CREDENTIALS_DIRECTORY}`.
|
||||
#
|
||||
# ⚠️ Unconditional, and a missing source is a unit that refuses to
|
||||
# start. That is the intended shape rather than an oversight: this
|
||||
# forwarder has one way to deliver, so a boot where the secret has
|
||||
# not arrived yet is a collector that says so and retries, not one
|
||||
# that quietly exports into a 401. The restart is what closes the
|
||||
# first-boot window the host unit's own comment describes.
|
||||
# post-boot window the host unit's own comment describes.
|
||||
#
|
||||
# ⚠️ `Restart` is deliberately NOT set here, same as the sibling
|
||||
# collector in ./swarm-otel.nix: nixpkgs' own
|
||||
|
|
@ -4254,7 +4282,7 @@ in
|
|||
# is still delivering the secret, leaving it failed rather than
|
||||
# retrying.
|
||||
systemd.services.opentelemetry-collector.serviceConfig = {
|
||||
LoadCredential = [ "${forwarderCredentialId}:${forwarderSecretInContainer}" ];
|
||||
LoadCredential = [ "${forwarderCredentialId}:${forwarderSecret}" ];
|
||||
RestartSec = 15;
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -368,9 +368,32 @@ let
|
|||
e.client_secret_file == "\${env:CREDENTIALS_DIRECTORY}/oidc-client-secret"
|
||||
&&
|
||||
unit.LoadCredential == [
|
||||
"oidc-client-secret:/var/lib/swarm-bao-otel-oidc/swarm-bao-collector.secret"
|
||||
"oidc-client-secret:/run/swarm-bao-forwarder-oidc/swarm-bao-collector.secret"
|
||||
];
|
||||
}
|
||||
{
|
||||
# A copy on persistent disk outlives the secret's rotation and lands in
|
||||
# backups of the container's root. The file is deliberately not created
|
||||
# ahead of the fetch: a placeholder would start the forwarder and have
|
||||
# its pushes refused instead of leaving it down until the store answers.
|
||||
name = "the forwarder's client secret is written to the host's /run only, and the old copy removed";
|
||||
ok =
|
||||
let
|
||||
m = baoWithCollector;
|
||||
script = m.systemd.services.swarm-bao-forwarder-oidc.script;
|
||||
mount = m.containers.swarm-bao.bindMounts."/run/swarm-bao-forwarder-oidc";
|
||||
rules = m.systemd.tmpfiles.rules;
|
||||
in
|
||||
lib.hasInfix "atomic_write_secret 0400 root:root /run/swarm-bao-forwarder-oidc/swarm-bao-collector.secret" script
|
||||
&& !(lib.hasInfix "/var/lib/nixos-containers" script)
|
||||
&& !(lib.hasInfix "/var/lib/swarm-bao-otel-oidc" script)
|
||||
&& mount.hostPath == "/run/swarm-bao-forwarder-oidc"
|
||||
&& mount.isReadOnly
|
||||
&& lib.elem "d /run/swarm-bao-forwarder-oidc 0700 root root -" rules
|
||||
&& lib.elem "r /var/lib/nixos-containers/swarm-bao/var/lib/swarm-bao-otel-oidc/swarm-bao-collector.secret" rules
|
||||
&& !(lib.any (r: lib.hasPrefix "f /run/swarm-bao-forwarder-oidc" r) rules)
|
||||
&& lib.elem "systemd-tmpfiles-resetup.service" m.systemd.services."container@swarm-bao".after;
|
||||
}
|
||||
{
|
||||
# Same 403-not-a-miss property the grafana and matrix readers are pinned
|
||||
# for: the reader's grant is one path under `services`, so a secret
|
||||
|
|
|
|||
Loading…
Reference in a new issue