From 7539ef82f7e233f05286d24c223c8b894fb8fc77 Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 15 Oct 2026 20:43:10 +0200 Subject: [PATCH] swarm-otel: keep the collector's OIDC secret on the host's /run, not in the container's root swarm-bao-otel-oidc still reads the secret from bao under the collector's own leaf, but now writes it to /run/swarm-otel-oidc on the host (tmpfs), which the container bind-mounts read-only for the collector's LoadCredential. Before, it wrote it to /var/lib/nixos-containers/swarm-otel/var/lib/swarm-otel-oidc/.secret on persistent disk, where a superseded secret stayed after rotation and went into backups of the container root. Host tmpfiles now removes that old file. Rotation is unchanged: the ten-minute timer rewrites the file when the value differs and refresh_consumer restarts the collector. After a reboot the secret is fetched again; until the store answers the collector starts on an empty file and the stores refuse its pushes and scrapes. Part of #4998 (slice 2a). --- docs/swarm/secrets.md | 2 +- nix/host-modules/swarm-otel.nix | 60 +++++++++++++++++++++-------- nix/module-eval/swarm-otel-core.nix | 26 +++++++++++++ 3 files changed, 71 insertions(+), 17 deletions(-) diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index 2eca677d..6ed124d8 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -51,7 +51,7 @@ neither is a renaming of the other. | authelia OIDC issuer key (RSA) | same unit | `/var/lib/authelia-swarm/oidc-issuer.key` | same — relying parties verify against the **public** half at `/jwks.json` | | OIDC client secret, plaintext half | `authelia crypto hash generate --random` | `/var/lib/authelia-swarm/oidc-clients/.secret` | operator provides the file and names it in whichever option reads it — `sso.clientSecretFile` for a service; the forge and, by default, the hive collector need none — `swarm-bao-forge-oidc` and `swarm-bao-hive-otel-oidc` fetch it from the store (below) | | OIDC client secret, digest half | the same mint | `oidc-clients/.digest` | authelia's own half; merged at runtime via `settingsFiles` | -| the swarm collector's copy of its OIDC secret | `swarm-bao-otel-oidc.service` reads it out of the swarm secret store, **on every host that runs the collector and holds a store identity** | `/var/lib/swarm-otel-oidc/.secret` inside the `swarm-otel` container | same unit, same path — one route, co-located or not. A collector with no store identity has `clientSecretFile == null`, its already-supported unauthenticated-push degrade — see below | +| the swarm collector's copy of its OIDC secret | `swarm-bao-otel-oidc.service` reads it out of the swarm secret store, **on every host that runs the collector and holds a store identity** | `/run/swarm-otel-oidc/.secret` on the host's tmpfs, bind-mounted read-only into the `swarm-otel` container — never on persistent disk, so a reboot fetches it again | same unit, same path — one route, co-located or not. A collector with no store identity has `clientSecretFile == null`, its already-supported unauthenticated-push degrade — see below | | Grafana's copy of its OIDC secret | `swarm-bao-grafana-oidc.service` reads it out of the swarm secret store, **on every host that runs Grafana** | `/var/lib/grafana-oidc/.secret` inside the `swarm-grafana` container | same unit, same path — one route, co-located or not. Nothing for an operator to place beyond this host's store leaf, see below | | Grafana's admin password | `swarm-bao-grafana-admin.service` creates it in the swarm secret store at `swarm/services//admin/password` when the store holds none, then copies it out, **on every host that runs Grafana**. Grafana's leaf may create that leaf but not update it, so the store keeps the first value, and the secret publisher can't write it. To rotate, run `bao kv metadata delete secret/swarm/services//admin/password` with a token that may, such as the root token (Grafana's, the publisher's and the controller's can't), then `systemctl restart swarm-bao-grafana-admin`; a plain `bao kv delete` isn't enough, see [the callout xkey's section](#the-queues-callout-xkey) | `/var/lib/grafana-admin/password` inside the `swarm-grafana` container: `security.admin_password` for a new database, and `grafana cli admin reset-admin-password` in `grafana.service`'s `preStart` for an existing one, on every start | same unit, same path. Grafana doesn't start until the file exists, so it stays down rather than falling back to the upstream default `admin` | | Grafana's metrics bearer | `swarm-bao-grafana-metrics-bearer.service` creates it in the swarm secret store at `swarm/services//metrics/bearer` when the store holds none, then copies it out, **on every host that runs Grafana**. Same grant as the admin password, create without update. To rotate, run `bao kv metadata delete secret/swarm/services//metrics/bearer` with a token that may, then `systemctl restart swarm-bao-grafana-metrics-bearer` | `/var/lib/grafana-metrics/bearer` inside the `swarm-grafana` container, holding `Bearer `: the metrics datasource's `Authorization` header. `swarm-metrics-auth` on the metrics store's host reads the stored value itself, under its own leaf | same unit, same path. The unit restarts Grafana when the file changes | diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index f638b52d..b1cf1037 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -140,18 +140,21 @@ let # re-exposes it to the dynamic user under a path that does not depend on # which uid it turned out to be. # - # At rest in the container's tree — written by the host oneshot below. - # Under /var/lib and not /run because the collector may start before the - # delivery unit on a later boot, and a secret that evaporates on reboot - # turns a working scrape into an intermittent one. - collectorSecretInContainer = "/var/lib/swarm-otel-oidc/${cfg.clientId}.secret"; - # The same file seen from the host, which is where the delivery unit - # writes it — spelled once for the same reason ./swarm-grafana.nix spells - # its own pair once: the unit that writes it and the option that names it - # are a few hundred lines apart, and a collector reading a path nothing + # On the host's /run (tmpfs), so the secret never reaches persistent disk: + # after a reboot the delivery unit below fetches it again. The container + # sees the directory read-only at the same path. The file starts out empty + # (tmpfiles, below) so `LoadCredential` has a source before the first + # fetch; a collector that starts on it is refused by the stores until the + # fetched secret restarts it. + # + # Spelled once, for the unit that writes it and the option that names it, + # which are a few hundred lines apart: a collector reading a path nothing # writes is an export that fails with nothing in any log about the file. - collectorHostSecretPath = "/var/lib/nixos-containers/${cfg.machine}${collectorSecretInContainer}"; - collectorHostSecretDir = builtins.dirOf collectorHostSecretPath; + collectorSecretDir = "/run/swarm-otel-oidc"; + collectorSecretInContainer = "${collectorSecretDir}/${cfg.clientId}.secret"; + # Where the secret sat on persistent disk before it moved to /run. + collectorLegacySecretInContainer = "/var/lib/swarm-otel-oidc/${cfg.clientId}.secret"; + collectorLegacyHostSecretPath = "/var/lib/nixos-containers/${cfg.machine}${collectorLegacySecretInContainer}"; collectorCredentialId = "oidc-client-secret"; # What the scrape config points at. ⚠️ This path and the `LoadCredential` # id below are one fact spelled twice by systemd's design — both derive from @@ -472,6 +475,11 @@ in ${atomicWriteSecret} ${refreshConsumer} + # The container's bind source, so it starts even when the store does + # not answer; tmpfiles creates it too, but unordered with the + # container on a switch. + install -d -m 0700 ${lib.escapeShellArg collectorSecretDir} + # `bao`'s own message is the only thing separating a missing value # from a refused identity from an unreachable host. This unit's # degraded mode is correct for all three, so it reports which one @@ -516,14 +524,13 @@ in # `path` would. The collector runs under `DynamicUser`, so there is # no uid to give this to — `LoadCredential` reads it as root before # the sandbox exists and re-exposes it to whichever uid the unit got. - install -d -m 0755 ${lib.escapeShellArg collectorHostSecretDir} - if secret_differs ${lib.escapeShellArg collectorHostSecretPath} "$secret"; then - atomic_write_secret 0400 root:root ${lib.escapeShellArg collectorHostSecretPath} "$secret" + if secret_differs ${lib.escapeShellArg collectorSecretInContainer} "$secret"; then + atomic_write_secret 0400 root:root ${lib.escapeShellArg collectorSecretInContainer} "$secret" fi # `LoadCredential` copies the file at start only, and a collector that # started without it is in its start limit. - refresh_consumer ${lib.escapeShellArg cfg.machine} opentelemetry-collector.service ${lib.escapeShellArg collectorHostSecretPath} + refresh_consumer ${lib.escapeShellArg cfg.machine} opentelemetry-collector.service ${lib.escapeShellArg collectorSecretInContainer} ''; }; @@ -540,6 +547,20 @@ in lib.mkDefault collectorSecretInContainer ); + systemd.tmpfiles.rules = lib.optionals haveClientIdentity ( + [ + # The bind source must exist when the container starts. + "d ${collectorSecretDir} 0700 root root -" + # `LoadCredential=` fails the unit when the file is missing; an empty + # one lets the collector start and be refused by the stores. + "f ${collectorSecretInContainer} 0400 root root -" + ] + # Unless a deployment delivers its own secret to that very path. + ++ lib.optional ( + deployCfg.swarm-otel.clientSecretFile != collectorLegacySecretInContainer + ) "r ${collectorLegacyHostSecretPath}" + ); + # The CA bind source is written at runtime by a host unit, so the # container has to start after it — otherwise nspawn sets up a mount # over a file that does not exist yet. @@ -788,7 +809,14 @@ in # does not exist makes nixos-container refuse to start the container, # which is a stall several layers from its cause. bindMounts = - lib.optionalAttrs (otelCfg.headersCredential != null) { + # The collector's OIDC secret, where the delivery unit writes it. + lib.optionalAttrs haveClientIdentity { + ${collectorSecretDir} = { + hostPath = collectorSecretDir; + isReadOnly = true; + }; + } + // lib.optionalAttrs (otelCfg.headersCredential != null) { ${otelCfg.headersCredential} = { hostPath = otelCfg.headersCredential; isReadOnly = true; diff --git a/nix/module-eval/swarm-otel-core.nix b/nix/module-eval/swarm-otel-core.nix index 4c4c6194..34a04165 100644 --- a/nix/module-eval/swarm-otel-core.nix +++ b/nix/module-eval/swarm-otel-core.nix @@ -105,6 +105,32 @@ let lib.hasInfix "secret/swarm/services/swarm-collector/oidc/client" s && !(lib.hasInfix "secret/swarm/hives/" s); } + { + # A copy on persistent disk outlives the secret's rotation and lands in + # backups of the container's root. + name = "the collector's OIDC secret is written to the host's /run only, and the old copy removed"; + ok = + let + m = otelBaoRemoteAuthelia; + script = m.systemd.services.swarm-bao-otel-oidc.script; + mount = m.containers.swarm-otel.bindMounts."/run/swarm-otel-oidc"; + rules = m.systemd.tmpfiles.rules; + in + lib.hasInfix "atomic_write_secret 0400 root:root /run/swarm-otel-oidc/swarm-collector.secret" script + && !(lib.hasInfix "/var/lib/nixos-containers" script) + && !(lib.hasInfix "/var/lib/swarm-otel-oidc" script) + && mount.hostPath == "/run/swarm-otel-oidc" + && mount.isReadOnly + && + m.services.hyperhive.deploy.swarm-otel.clientSecretFile + == "/run/swarm-otel-oidc/swarm-collector.secret" + && lib.elem "d /run/swarm-otel-oidc 0700 root root -" rules + && lib.elem "f /run/swarm-otel-oidc/swarm-collector.secret 0400 root root -" rules + && lib.elem "r /var/lib/nixos-containers/swarm-otel/var/lib/swarm-otel-oidc/swarm-collector.secret" rules + && + lib.any (c: c == "oidc-client-secret:/run/swarm-otel-oidc/swarm-collector.secret") + m.containers.swarm-otel.config.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential; + } { # These exporters must not be gated on the stores' PER-HOST enables: a # collector that did not share a host with them would render none at all