diff --git a/docs/swarm/sso.md b/docs/swarm/sso.md index 133a3067..ed88b73b 100644 --- a/docs/swarm/sso.md +++ b/docs/swarm/sso.md @@ -316,6 +316,9 @@ gateway refuses its token too. See [`swarmctl/README.md`](../../swarmctl/README. and calls the API with it. See [`services.md`](services.md#metrics-victoriametrics--grafana) and [`secrets.md`](secrets.md). SSO logins are the everyday way in, which is why `swarm.grafana.oidc.role` defaults to `Admin`. + The three secrets Grafana reads live on the host's tmpfs, so after a reboot + Grafana stays down (no login, no dashboards) until the secret store + answers; the delivery units retry and start it with no manual step. - **It doesn't provision users.** `swarmctl` creates and destroys agents continuously today, so the subject set belongs to a program rather than to a config file. diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 8e878a19..dbbe9c37 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -299,6 +299,12 @@ let atomicWriteSecret = import ./lib/atomic-write-secret.nix { }; refreshConsumer = import ./lib/refresh-consumer.nix { }; + # First statement of each delivery script, ahead of the store login: the + # directory is the container's bind source, so it has to exist even when the + # store does not answer. tmpfiles creates it too, but unordered with the + # container on a switch. + ensureSecretDir = "install -d -m 0755 ${lib.escapeShellArg secretDir}"; + # Shared host netns, like every sibling swarm container: the gateway # reaches this at 127.0.0.1:. privateNetwork = false; @@ -554,10 +560,11 @@ in # # The secrets directory is the container's read-only bind source, so it # has to exist before the container starts even when the store does not - # answer; the delivery units create it too, but unordered with the - # container on a switch. 0755 because Grafana reads the files as its own - # uid; each file is 0400 and owned by it. The legacy files are removed so - # no superseded secret stays behind on the host's disk. + # answer. The delivery units create it before their store login too + # (`ensureSecretDir`), which covers a switch, where this rule is unordered + # with the container restart. 0755 because Grafana reads the files as its + # own uid; each file is 0400 and owned by it. The legacy files are removed + # so no superseded secret stays behind on the host's disk. systemd.tmpfiles.rules = [ "d ${deployCfg.grafana.socketDir} 0750 ${toString grafanaUid} ${toString nginxGid} - -" "d ${secretDir} 0755 root root -" @@ -638,6 +645,8 @@ in ${atomicWriteSecret} ${refreshConsumer} + ${ensureSecretDir} + # `bao`'s own message is the only thing separating a missing value from # a refused identity from an unreachable host. This unit's degraded # mode is correct for all three, so it reports which one rather than @@ -693,7 +702,6 @@ in # the host cannot know it at eval time. Harmless here because 0400 # grants the group nothing; if this mode ever widens, the gid has to be # discovered at runtime rather than assumed. - install -d -m 0755 ${lib.escapeShellArg secretDir} if secret_differs ${lib.escapeShellArg secretPath} "$secret"; then atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg secretPath} "$secret" fi @@ -750,6 +758,8 @@ in ${atomicWriteSecret} ${refreshConsumer} + ${ensureSecretDir} + err="$(mktemp)" trap 'rm -f "$err"' EXIT @@ -781,7 +791,6 @@ in fi # Owner and mode as for the OIDC secret above, for its reasons. - install -d -m 0755 ${lib.escapeShellArg secretDir} if secret_differs ${lib.escapeShellArg adminPasswordPath} "$password"; then atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg adminPasswordPath} "$password" fi @@ -832,6 +841,8 @@ in ${atomicWriteSecret} ${refreshConsumer} + ${ensureSecretDir} + err="$(mktemp)" trap 'rm -f "$err"' EXIT @@ -862,7 +873,6 @@ in fi # Owner and mode as for the OIDC secret above, for its reasons. - install -d -m 0755 ${lib.escapeShellArg secretDir} if secret_differs ${lib.escapeShellArg metricsBearerPath} "Bearer $bearer"; then atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg metricsBearerPath} "Bearer $bearer" fi diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix index 325db82e..06d3d568 100644 --- a/nix/module-eval/grafana.nix +++ b/nix/module-eval/grafana.nix @@ -305,9 +305,19 @@ let oidc = services.swarm-bao-grafana-oidc.script; admin = services.swarm-bao-grafana-admin.script; bearer = services.swarm-bao-grafana-metrics-bearer.script; + # The bind source must exist when the store refuses the login, so the + # directory is made ahead of it in every delivery script. + dirBeforeLogin = + s: + lib.hasInfix "install -d -m 0755 /run/swarm-grafana-secrets" ( + lib.head (lib.splitString "bao login -method=cert" s) + ); in lib.hasInfix "atomic_write_secret 0400 196:0 /run/swarm-grafana-secrets/swarm-grafana.secret" oidc && lib.hasInfix "atomic_write_secret 0400 196:0 /run/swarm-grafana-secrets/metrics-bearer" bearer + && dirBeforeLogin oidc + && dirBeforeLogin admin + && dirBeforeLogin bearer && !(persistent oidc) && !(persistent admin) && !(persistent bearer)