vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-grafana: keep Grafana's OIDC secret, admin password and metrics bearer on the host's /run, not in the container's root

Browse source 
The three delivery units still read from bao under Grafana's own leaf, but
now write to /run/swarm-grafana-secrets on the host (tmpfs), which the
container bind-mounts read-only at the same path. Before, they wrote to
/var/lib/nixos-containers/swarm-grafana/var/lib/{grafana-oidc,grafana-admin,
grafana-metrics}/ on persistent disk, where a superseded value stayed after
rotation and went into backups of the container root. Host tmpfiles now
removes the three old files.

Rotation is unchanged: refresh_consumer restarts Grafana (or starts it if it
failed) when a file changes, since Grafana reads each $__file{} once as it
starts. After a reboot the files are fetched again; until the store answers
Grafana stays down, where before it came up on the last copy from disk.

Part of #4998 (slice 2b).
This commit is contained in:
atlas 2026-10-15 22:09:03 +02:00 • committed by mara
commit cbab23273a
4 changed files with 95 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

81
nix/host-modules/swarm-grafana.nix
View file

@ -226,18 +226,26 @@ let
autheliaUrl = toString hyperhiveCfg.swarm.authelia.url;
# Where the plaintext lands inside the container. Under /var/lib rather
# than /run: Grafana may start before the delivery unit on a later boot,
# and a secret that evaporates on reboot turns a working login into an
# intermittent one.
secretPath = "/var/lib/grafana-oidc/${cfg.oidc.clientId}.secret";
# The same file seen from the host, which is where the delivery unit writes
# it. Spelled once: the unit that writes it and the config that names it are
# The three secrets below land on the host's /run (tmpfs), which the
# container bind-mounts read-only at the same path: none reaches persistent
# disk, and a reboot fetches them again. Each file is absent until its
# delivery unit has run, and Grafana does not start without them — `$__file{}`
# on a missing path is a startup error, and an empty admin password would
# fall back to the upstream default. A delivery unit that writes a file
# restarts or starts Grafana (`refresh_consumer`).
#
# Spelled once: the unit that writes a file and the config that names it are
# a few hundred lines apart, and a Grafana reading a path nothing writes is
# a login that fails with nothing in any log about the file.
hostSecretPath = "/var/lib/nixos-containers/${cfg.machine}${secretPath}";
hostSecretDir = builtins.dirOf hostSecretPath;
secretDir = "/run/swarm-grafana-secrets";
secretPath = "${secretDir}/${cfg.oidc.clientId}.secret";
# Where the three files sat on persistent disk before they moved to /run.
legacyHostSecretPaths = map (p: "/var/lib/nixos-containers/${cfg.machine}${p}") [
"/var/lib/grafana-oidc/${cfg.oidc.clientId}.secret"
"/var/lib/grafana-admin/password"
"/var/lib/grafana-metrics/bearer"
];
# Where the publisher on authelia's host leaves this client's secret. A
# swarm service's client is registered once for the whole swarm, so its
@ -260,8 +268,7 @@ let
# store by `swarm-bao-grafana-admin` below, which `swarm-bao.nix`'s
# `grafanaOidcReaders` grants create and read on this one leaf.
adminPasswordStorePath = "secret/swarm/services/${cfg.oidc.clientId}/admin/password";
adminPasswordPath = "/var/lib/grafana-admin/password";
hostAdminPasswordPath = "/var/lib/nixos-containers/${cfg.machine}${adminPasswordPath}";
adminPasswordPath = "${secretDir}/admin-password";
# The bearer the metrics datasource presents at the metrics store's query
# API, which `swarm-metrics-auth` on the store's host checks against the
@ -270,8 +277,7 @@ let
# header value, `Bearer <value>`: nixpkgs accepts a `secureJsonData` entry
# only when it is exactly one `$__file{}`.
metricsBearerStorePath = "secret/swarm/services/${cfg.oidc.clientId}/metrics/bearer";
metricsBearerPath = "/var/lib/grafana-metrics/bearer";
hostMetricsBearerPath = "/var/lib/nixos-containers/${cfg.machine}${metricsBearerPath}";
metricsBearerPath = "${secretDir}/metrics-bearer";
# swarm-controller's token for Grafana's API, minted by
# `swarm-bao-grafana-controller-token` below for a Viewer service account of
@ -545,9 +551,18 @@ in
# gateway can traverse; `0750` keeps everything else out. Numeric ids
# because the host has no `grafana` account — Grafana lives in the
# container, and only the number crosses that boundary.
#
# The secrets directory is the container's read-only bind source, so it
# has to exist before the container starts even when the store does not
# answer; the delivery units create it too, but unordered with the
# container on a switch. 0755 because Grafana reads the files as its own
# uid; each file is 0400 and owned by it. The legacy files are removed so
# no superseded secret stays behind on the host's disk.
systemd.tmpfiles.rules = [
"d ${deployCfg.grafana.socketDir} 0750 ${toString grafanaUid} ${toString nginxGid} - -"
];
"d ${secretDir} 0755 root root -"
]
++ map (p: "r ${p}") legacyHostSecretPaths;
# THE delivery unit — one route, in every deployment. The secret authelia
# minted arrives out of the swarm secret store, which the publisher on
@ -678,13 +693,13 @@ in
# the host cannot know it at eval time. Harmless here because 0400
# grants the group nothing; if this mode ever widens, the gid has to be
# discovered at runtime rather than assumed.
install -d -m 0755 ${lib.escapeShellArg hostSecretDir}
if secret_differs ${lib.escapeShellArg hostSecretPath} "$secret"; then
atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg hostSecretPath} "$secret"
install -d -m 0755 ${lib.escapeShellArg secretDir}
if secret_differs ${lib.escapeShellArg secretPath} "$secret"; then
atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg secretPath} "$secret"
fi
# `$__file{}` is expanded once, while Grafana parses its config.
refresh_consumer ${lib.escapeShellArg cfg.machine} grafana.service ${lib.escapeShellArg hostSecretPath}
refresh_consumer ${lib.escapeShellArg cfg.machine} grafana.service ${lib.escapeShellArg secretPath}
'';
};
@ -766,14 +781,14 @@ in
fi
# Owner and mode as for the OIDC secret above, for its reasons.
install -d -m 0755 ${lib.escapeShellArg (builtins.dirOf hostAdminPasswordPath)}
if secret_differs ${lib.escapeShellArg hostAdminPasswordPath} "$password"; then
atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg hostAdminPasswordPath} "$password"
install -d -m 0755 ${lib.escapeShellArg secretDir}
if secret_differs ${lib.escapeShellArg adminPasswordPath} "$password"; then
atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg adminPasswordPath} "$password"
fi
# Grafana's `preStart` applies the file to the database, so a file that
# changed after it started needs a restart.
refresh_consumer ${lib.escapeShellArg cfg.machine} grafana.service ${lib.escapeShellArg hostAdminPasswordPath}
refresh_consumer ${lib.escapeShellArg cfg.machine} grafana.service ${lib.escapeShellArg adminPasswordPath}
'';
};
@ -847,13 +862,13 @@ in
fi
# Owner and mode as for the OIDC secret above, for its reasons.
install -d -m 0755 ${lib.escapeShellArg (builtins.dirOf hostMetricsBearerPath)}
if secret_differs ${lib.escapeShellArg hostMetricsBearerPath} "Bearer $bearer"; then
atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg hostMetricsBearerPath} "Bearer $bearer"
install -d -m 0755 ${lib.escapeShellArg secretDir}
if secret_differs ${lib.escapeShellArg metricsBearerPath} "Bearer $bearer"; then
atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg metricsBearerPath} "Bearer $bearer"
fi
# `$__file{}` in a provisioned datasource is read as Grafana starts.
refresh_consumer ${lib.escapeShellArg cfg.machine} grafana.service ${lib.escapeShellArg hostMetricsBearerPath}
refresh_consumer ${lib.escapeShellArg cfg.machine} grafana.service ${lib.escapeShellArg metricsBearerPath}
'';
};
@ -935,11 +950,11 @@ in
exit 1
}
if [ ! -s ${lib.escapeShellArg hostAdminPasswordPath} ]; then
echo "no Grafana admin password at ${hostAdminPasswordPath} yet; swarm-bao-grafana-admin writes it." >&2
if [ ! -s ${lib.escapeShellArg adminPasswordPath} ]; then
echo "no Grafana admin password at ${adminPasswordPath} yet; swarm-bao-grafana-admin writes it." >&2
exit 1
fi
admin="Basic $(printf 'admin:%s' "$(< ${lib.escapeShellArg hostAdminPasswordPath})" | base64 -w0)"
admin="Basic $(printf 'admin:%s' "$(< ${lib.escapeShellArg adminPasswordPath})" | base64 -w0)"
status="$(grafana "$admin" GET '/api/serviceaccounts/search?perpage=1000&query=${controllerAccountName}')"
expect "the service account search" "$status" 200
@ -1020,6 +1035,12 @@ in
hostPath = deployCfg.grafana.socketDir;
isReadOnly = false;
};
# The three delivered secrets, read-only: the container reads them and
# never writes one.
${secretDir} = {
hostPath = secretDir;
isReadOnly = true;
};
}
// caTrust.bindMount;