grafana: create the secrets directory before the store login

The three delivery units made /run/swarm-grafana-secrets only after
`bao login` and `bao kv get`, so with the store sealed or unreachable the
bind source was never created by them and a switch could restart the
container without it. Each script now starts with the install, from one
shared `ensureSecretDir`, and the tmpfiles comment says what is true.

The module-eval case asserts the install precedes `bao login` in all three
scripts. docs/swarm/sso.md states that after a reboot Grafana stays down
until the secret store answers.

Part of #4998 (slice 2b: grafana).

docs/swarm/sso.md

@@ -316,6 +316,9 @@ gateway refuses its token too. See [`swarmctl/README.md`](../../swarmctl/README.
   and calls the API with it. See [`services.md`](services.md#metrics-victoriametrics--grafana)
   and [`secrets.md`](secrets.md). SSO logins are the everyday way in, which
   is why `swarm.grafana.oidc.role` defaults to `Admin`.
+  The three secrets Grafana reads live on the host's tmpfs, so after a reboot
+  Grafana stays down (no login, no dashboards) until the secret store
+  answers; the delivery units retry and start it with no manual step.
 - **It doesn't provision users.** `swarmctl` creates and destroys agents
   continuously today, so the subject set belongs to a program rather
   than to a config file.

nix/host-modules/swarm-grafana.nix

@@ -299,6 +299,12 @@ let
   atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
   refreshConsumer = import ./lib/refresh-consumer.nix { };
 
+  # First statement of each delivery script, ahead of the store login: the
+  # directory is the container's bind source, so it has to exist even when the
+  # store does not answer. tmpfiles creates it too, but unordered with the
+  # container on a switch.
+  ensureSecretDir = "install -d -m 0755 ${lib.escapeShellArg secretDir}";
+
   # Shared host netns, like every sibling swarm container: the gateway
   # reaches this at 127.0.0.1:<port>.
   privateNetwork = false;
@@ -554,10 +560,11 @@ in
     #
     # The secrets directory is the container's read-only bind source, so it
     # has to exist before the container starts even when the store does not
-    # answer; the delivery units create it too, but unordered with the
+    # answer. The delivery units create it before their store login too
-    # container on a switch. 0755 because Grafana reads the files as its own
+    # (`ensureSecretDir`), which covers a switch, where this rule is unordered
-    # uid; each file is 0400 and owned by it. The legacy files are removed so
+    # with the container restart. 0755 because Grafana reads the files as its
-    # no superseded secret stays behind on the host's disk.
+    # own uid; each file is 0400 and owned by it. The legacy files are removed
+    # so no superseded secret stays behind on the host's disk.
     systemd.tmpfiles.rules = [
       "d ${deployCfg.grafana.socketDir} 0750 ${toString grafanaUid} ${toString nginxGid} - -"
       "d ${secretDir} 0755 root root -"
@@ -638,6 +645,8 @@ in
         ${atomicWriteSecret}
         ${refreshConsumer}
 
+        ${ensureSecretDir}
+
         # `bao`'s own message is the only thing separating a missing value from
         # a refused identity from an unreachable host. This unit's degraded
         # mode is correct for all three, so it reports which one rather than
@@ -693,7 +702,6 @@ in
         # the host cannot know it at eval time. Harmless here because 0400
         # grants the group nothing; if this mode ever widens, the gid has to be
         # discovered at runtime rather than assumed.
-        install -d -m 0755 ${lib.escapeShellArg secretDir}
         if secret_differs ${lib.escapeShellArg secretPath} "$secret"; then
           atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg secretPath} "$secret"
         fi
@@ -750,6 +758,8 @@ in
         ${atomicWriteSecret}
         ${refreshConsumer}
 
+        ${ensureSecretDir}
+
         err="$(mktemp)"
         trap 'rm -f "$err"' EXIT
 
@@ -781,7 +791,6 @@ in
         fi
 
         # Owner and mode as for the OIDC secret above, for its reasons.
-        install -d -m 0755 ${lib.escapeShellArg secretDir}
         if secret_differs ${lib.escapeShellArg adminPasswordPath} "$password"; then
           atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg adminPasswordPath} "$password"
         fi
@@ -832,6 +841,8 @@ in
         ${atomicWriteSecret}
         ${refreshConsumer}
 
+        ${ensureSecretDir}
+
         err="$(mktemp)"
         trap 'rm -f "$err"' EXIT
 
@@ -862,7 +873,6 @@ in
         fi
 
         # Owner and mode as for the OIDC secret above, for its reasons.
-        install -d -m 0755 ${lib.escapeShellArg secretDir}
         if secret_differs ${lib.escapeShellArg metricsBearerPath} "Bearer $bearer"; then
           atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg metricsBearerPath} "Bearer $bearer"
         fi

nix/module-eval/grafana.nix

@@ -305,9 +305,19 @@ let
           oidc = services.swarm-bao-grafana-oidc.script;
           admin = services.swarm-bao-grafana-admin.script;
           bearer = services.swarm-bao-grafana-metrics-bearer.script;
+          # The bind source must exist when the store refuses the login, so the
+          # directory is made ahead of it in every delivery script.
+          dirBeforeLogin =
+            s:
+            lib.hasInfix "install -d -m 0755 /run/swarm-grafana-secrets" (
+              lib.head (lib.splitString "bao login -method=cert" s)
+            );
         in
         lib.hasInfix "atomic_write_secret 0400 196:0 /run/swarm-grafana-secrets/swarm-grafana.secret" oidc
         && lib.hasInfix "atomic_write_secret 0400 196:0 /run/swarm-grafana-secrets/metrics-bearer" bearer
+        && dirBeforeLogin oidc
+        && dirBeforeLogin admin
+        && dirBeforeLogin bearer
         && !(persistent oidc)
         && !(persistent admin)
         && !(persistent bearer)