Watch
0
0
Fork
You've already forked hyperhive
0

keep each hive's otel oidc secret on /run, not /var/lib

This commit is contained in:
atlas 2026-10-25 11:05:47 +01:00 • committed by mara
commit 96405621d7
7 changed files with 106 additions and 18 deletions

View file

@ -30,6 +30,11 @@ let
baoCfg = hyperhiveCfg.swarm.bao;
otel = hyperhiveCfg.otel;
hiveOtelSecret = import ./lib/hive-otel-secret.nix {
inherit (hyperhiveCfg.swarm.authelia) hiveClientPrefix;
inherit (hyperhiveCfg) hiveName;
};
havePair = cert: key: cert != null && key != null;
# Reader unit → the gate its own module defines it under.
@ -44,9 +49,7 @@ let
otel.collector.enable
&& hyperhiveCfg.hiveName != null
&& havePair baoDeploy.clientCertFile baoDeploy.clientKeyFile
&&
otel.clientSecretFile
== "/var/lib/hive-otel-oidc/${hyperhiveCfg.swarm.authelia.hiveClientPrefix}${toString hyperhiveCfg.hiveName}.secret";
&& otel.clientSecretFile == hiveOtelSecret.path;
# ./swarm-grafana.nix
swarm-bao-grafana-oidc = deployCfg.grafana.enable;
swarm-bao-grafana-admin = deployCfg.grafana.enable;

View file

@ -26,7 +26,11 @@ let
senderAuth = otel.clientSecretFile != null && hyperhiveCfg.hiveName != null;
hiveLeaf =
hyperhiveCfg.deploy.bao.clientCertFile != null && hyperhiveCfg.deploy.bao.clientKeyFile != null;
fetchedSecretPath = "/var/lib/hive-otel-oidc/${hyperhiveCfg.swarm.authelia.hiveClientPrefix}${toString hyperhiveCfg.hiveName}.secret";
fetchedSecretPath =
(import ./lib/hive-otel-secret.nix {
inherit (hyperhiveCfg.swarm.authelia) hiveClientPrefix;
inherit (hyperhiveCfg) hiveName;
}).path;
fetches = senderAuth && hiveLeaf && otel.clientSecretFile == fetchedSecretPath;
waits = senderAuth && !fetches;
in

View file

@ -0,0 +1,22 @@
# Where `swarm-bao-hive-otel-oidc` lands this hive's OIDC client secret on
# the host, and where it sat before it moved. Spelled once because
# ../otel.nix, ../glue-hive-otel-authelia.nix and
# ../glue-bao-readers-store-order.nix each compare `otel.clientSecretFile`
# against it to decide whether the fetch unit exists; a copy that drifts makes
# that comparison false and the secret stops refreshing.
#
# `path` is on /run (tmpfs): the secret never reaches persistent disk and the
# unit fetches it again after a reboot.
#
# Pure function — NOT a NixOS module. Call it from a module's `let`:
#
# hiveOtelSecret = import ./lib/hive-otel-secret.nix { inherit (cfg) hiveClientPrefix hiveName; };
{ hiveClientPrefix, hiveName }:
let
file = "${hiveClientPrefix}${toString hiveName}.secret";
in
{
dir = "/run/hive-otel-oidc";
path = "/run/hive-otel-oidc/${file}";
legacyPath = "/var/lib/hive-otel-oidc/${file}";
}

View file

@ -46,7 +46,11 @@ let
&& config.services.hyperhive.deploy.bao.clientKeyFile != null;
# Where `swarm-bao-hive-otel-oidc` lands the secret on this host.
fetchedSecretPath = "/var/lib/hive-otel-oidc/${config.services.hyperhive.swarm.authelia.hiveClientPrefix}${toString config.services.hyperhive.hiveName}.secret";
hiveOtelSecret = import ./lib/hive-otel-secret.nix {
inherit (config.services.hyperhive.swarm.authelia) hiveClientPrefix;
inherit (config.services.hyperhive) hiveName;
};
fetchedSecretPath = hiveOtelSecret.path;
in
{
imports = [
@ -188,7 +192,7 @@ in
type = lib.types.nullOr lib.types.str;
default =
if hiveLeaf && config.services.hyperhive.hiveName != null then fetchedSecretPath else null;
defaultText = lib.literalExpression ''"/var/lib/hive-otel-oidc/''${swarm.authelia.hiveClientPrefix}''${hiveName}.secret" when deploy.bao.clientCertFile and clientKeyFile are set, else null'';
defaultText = lib.literalExpression ''"/run/hive-otel-oidc/''${swarm.authelia.hiveClientPrefix}''${hiveName}.secret" when deploy.bao.clientCertFile and clientKeyFile are set, else null'';
example = "/var/lib/secrets/hive-telemetry.secret";
description = ''
Absolute path to this hive's OAuth2 client secret, used to
@ -830,7 +834,7 @@ in
# Root-owned 0400: the collector runs under `DynamicUser`, so there is
# no stable uid to hand the file to, and `LoadCredential` reads it as
# root before the sandbox exists.
install -d -m 0700 ${lib.escapeShellArg (builtins.dirOf fetchedSecretPath)}
install -d -m 0700 ${lib.escapeShellArg hiveOtelSecret.dir}
if secret_differs ${lib.escapeShellArg fetchedSecretPath} "$secret"; then
atomic_write_secret 0400 "" ${lib.escapeShellArg fetchedSecretPath} "$secret"
fi
@ -843,6 +847,29 @@ in
storeRefreshTimer "re-read this hive's telemetry OIDC client secret from the swarm secret store"
);
systemd.tmpfiles.rules =
lib.optionals fetches [
"d ${hiveOtelSecret.dir} 0700 root root -"
# `LoadCredential=` fails the unit when the file is missing; an empty
# one lets the collector start and be refused by the stores until the
# fetch lands.
"f ${fetchedSecretPath} 0400 root root -"
]
# Unless a deployment names that very path itself.
++ lib.optional (
hiveName != null && otel.clientSecretFile != hiveOtelSecret.legacyPath
) "r ${hiveOtelSecret.legacyPath}";
# The fetch unit writes `fetchedSecretPath` now, so a file named at the
# old default is delivered by nobody and never refreshed.
warnings = lib.optional (hiveName != null && otel.clientSecretFile == hiveOtelSecret.legacyPath) ''
services.hyperhive.otel.clientSecretFile is ${hiveOtelSecret.legacyPath}, the path
swarm-bao-hive-otel-oidc.service used to write. That unit now writes
${fetchedSecretPath} and, with the option set to anything else, does
not exist, so nothing refreshes the file. Remove the setting to fetch
from the swarm secret store, or deliver the file yourself.
'';
assertions = [
{
# A hive authenticates to the swarm's collector as itself — there