From a4e48a7bca3b576e44d68b25bc6b8a7c27f21e51 Mon Sep 17 00:00:00 2001
From: atlas <atlas@hyperhive.local>
Date: Tue, 27 Oct 2026 12:02:44 +0100
Subject: [PATCH] store the forge webhook secret and grafana's secret key in
 bao, seeded from the existing files

---
 docs/agent-lifecycle/persistence.md           |  13 +-
 docs/swarm/credentials-reference.md           |  82 ++--
 .../glue-bao-readers-policy-order.nix         |   8 +-
 .../glue-bao-readers-store-order.nix          |   1 +
 nix/host-modules/swarm-bao.nix                |  22 +-
 nix/host-modules/swarm-grafana.nix            | 183 ++++++---
 nix/module-eval/bao-grants.nix                |  46 ++-
 nix/module-eval/grafana.nix                   |  31 ++
 nix/module-eval/swarm-services-switch.nix     |   5 +-
 swarm-controller/README.md                    |   4 +-
 swarm-controller/src/main.rs                  |   8 +-
 swarm-controller/src/startup.rs               |   4 +-
 swarm-controller/src/webhook.rs               | 367 ++++++++++++++++--
 swarm-secret-client/src/forge.rs              |  36 ++
 swarm-secret-client/src/lib.rs                |  29 ++
 15 files changed, 697 insertions(+), 142 deletions(-)

diff --git a/docs/agent-lifecycle/persistence.md b/docs/agent-lifecycle/persistence.md
index 99edfcbe..1838fed4 100644
--- a/docs/agent-lifecycle/persistence.md
+++ b/docs/agent-lifecycle/persistence.md
@@ -497,13 +497,16 @@ Only present on the one host running
 so it survives restarts and redeploys.
 
 - `webhook-secret` — the HMAC key Forgejo signs the swarm's forge webhooks
-  with. **Keep it.** It's handed to Forgejo when swarm-controller registers a hook,
-  so replacing the file means every subsequent delivery fails
-  verification until the hook is re-registered with the new value. It's
-  generated automatically on first start; there's nothing to configure.
+  with, as it was before the swarm secret store held it. **Keep it.** The
+  controller stores its value in the swarm secret store at
+  `swarm/controller/swarm-controller/forge/webhook-secret` the first time it
+  starts with that path empty, and reads the store from then on. The file is the
+  fallback while the store is unreachable or its policy doesn't grant the path
+  yet, and nothing else reads or removes it. A host with neither the file nor a
+  store value gets a new secret in the store, with nothing to configure.
 
 <!-- vale write-good.Passive = NO -->
-If the file is unreadable at startup the daemon still starts and logs
+If the store can't answer and the file is unreadable or missing at startup, the daemon still starts and logs
 `webhook secret unavailable`; the webhook endpoint then answers 503
 rather than accepting deliveries it can't verify. Everything else the
 controller serves is unaffected.
diff --git a/docs/swarm/credentials-reference.md b/docs/swarm/credentials-reference.md
index dadc9075..93349050 100644
--- a/docs/swarm/credentials-reference.md
+++ b/docs/swarm/credentials-reference.md
@@ -14,33 +14,35 @@ the rest of the cell says how.
 
 <!-- vale write-good.Passive = NO -->
 
-| store path                                                  | minter                                                                               | reader                                              | automatic re-mint                                                                                              | automatic re-pull                                                                                                                                                                                                                                            |
-| ----------------------------------------------------------- | ------------------------------------------------------------------------------------ | --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| `swarm/agents/<agent>/matrix/main`                          | `swarm-controller`, at agent creation and every 5 minutes                            | the agent                                           | ✅ when missing or rejected by the homeserver                                                                  | ✅ the daemon restarts within 5 minutes of a rejection                                                                                                                                                                                                       |
-| `swarm/agents/<agent>/matrix/<account>`                     | `swarm-controller`, from a credential you hand it                                    | the agent                                           | must be stated                                                                                                 | ✅ retried within 2 to 16 minutes after a rejection                                                                                                                                                                                                          |
-| `swarm/agents/<agent>/matrix-recovery/<account>`            | `swarm-controller`, a random passphrase per account                                  | the agent                                           | ❌ never replaced: the account's key backup opens only with this passphrase                                    | ❌ read when the account comes up                                                                                                                                                                                                                            |
-| `swarm/controller/swarm-controller/matrix/appservice-token` | `swarm-matrix-ctl` in `hive-matrix`, once                                            | `swarm-controller`                                  | ❌ minted once                                                                                                 | ✅ read every 5 minutes                                                                                                                                                                                                                                      |
-| `swarm/controller/swarm-controller/oidc/client`             | authelia, first boot                                                                 | `swarm-controller`                                  | ❌ minted once; `swarm-secret-publish` republishes a re-mint                                                   | ✅ re-read when authelia refuses it                                                                                                                                                                                                                          |
-| `swarm/controller/swarm-controller/forge/token`             | the forge, once                                                                      | `swarm-controller`                                  | ❌ minted once; delete `/var/lib/forgejo/swarm-controller-token-name` in the forge container to mint a new one | ✅ re-read on a 401                                                                                                                                                                                                                                          |
-| `swarm/controller/swarm-controller/grafana/token`           | `swarm-bao-grafana-controller-token` on Grafana's host                               | `swarm-controller`                                  | ✅ every 10 minutes, when Grafana refuses it or none is stored                                                 | ✅ re-read on a 401                                                                                                                                                                                                                                          |
-| `swarm/agents/<agent>/bao-mtls`                             | the store's agent PKI mount (`deploy.bao.agentPkiMountPath`), at agent creation      | `hive-c0re`, then the agent                         | ✅ re-issued every 5 minutes once past half its 7-day validity                                                 | ✅ the agent re-reads every 10 minutes; it renews only while its current leaf is valid                                                                                                                                                                       |
-| `swarm/agents/<agent>/queue`                                | `swarm-controller`, at agent creation                                                | the agent; the queue, log and metrics auth services | ✅ re-minted at 45 days; skipped for `Destroyed` agents, whose path is deleted                                 | ✅ read on every reconnect; an open connection keeps the old secret for up to an hour; the log and metrics stores honour a replaced secret for up to `deploy.victorialogs.authVerdictTtl` and `deploy.victoriametrics.agentAuthVerdictTtl` (60 s by default) |
-| `swarm/agents/<agent>/forge-token`                          | `swarm-controller`, at agent creation and every 5 minutes                            | the agent                                           | ✅ when missing or no longer matching the forge                                                                | ✅ every 10 minutes                                                                                                                                                                                                                                          |
-| `swarm/agents/<agent>/forge/<label>`                        | `swarm-controller`, when you link an external forge account in the swarm UI          | the agent                                           | ❌ your token; delete it and link the label again                                                              | ✅ every 2 minutes                                                                                                                                                                                                                                           |
-| `swarm/agents/<agent>/github-token`                         | `swarm-controller`, when you link a GitHub account in the swarm UI                   | the agent                                           | ❌ your token; delete it and link again                                                                        | ✅ every 2 minutes                                                                                                                                                                                                                                           |
-| `swarm/agents/<agent>/claude-credentials`                   | `swarm-claude-login`, when you log the agent in from its card; one version per login | the agent, while it waits for a login               | ❌ log in again                                                                                                | ✅ every 30 seconds while waiting                                                                                                                                                                                                                            |
-| `swarm/hives/<hive>/matrix/sender-token`                    | `swarm-controller`, every 5 minutes                                                  | `swarm-controller` and the hive                     | ✅ when missing or rejected by the homeserver                                                                  | ✅ the hive re-reads on every sweep                                                                                                                                                                                                                          |
-| `swarm/hives/<hive>/forge/token`                            | `swarm-controller`, every 5 minutes                                                  | `swarm-controller` and the hive                     | ✅ when missing or no longer matching the forge                                                                | ✅ read on every forge call                                                                                                                                                                                                                                  |
-| `swarm/hives/<hive>/oidc/client`                            | authelia, first boot; `swarm-secret-publish` copies it in                            | the hive                                            | ❌ minted once; `swarm-secret-publish` republishes a re-mint                                                   | ✅ re-read when authelia refuses it                                                                                                                                                                                                                          |
-| `swarm/services/<clientId>/oidc/client`                     | authelia                                                                             | that service                                        | must be stated                                                                                                 | must be stated                                                                                                                                                                                                                                               |
-| `swarm/services/ci-runner/registration-token`               | `swarm-controller`, from the forge, at start and every 5 minutes                     | `swarm-bao-ci-runner` on the runner's host          | ❌ the forge keeps one token until an admin resets it; the controller then writes the new one within 5 minutes | ✅ every 10 minutes; restarts the runner on a change                                                                                                                                                                                                         |
-| `swarm/services/<grafana client id>/metrics/bearer`         | `swarm-bao-grafana-metrics-bearer` on Grafana's host, when none is stored            | `swarm-metrics-auth` and `swarm-logs-grafana-auth`  | ❌ delete it and restart `swarm-bao-grafana-metrics-bearer`                                                    | ✅ the auth services re-read within a minute; ❌ Grafana restarts when its copy changes                                                                                                                                                                      |
-| `swarm/services/review-classifier/anthropic-api-key`        | you, in the store's UI, field `api_key`                                              | `swarm-review-classifier`                           | ❌ write a new key                                                                                             | ✅ read on every run                                                                                                                                                                                                                                         |
-| `swarm/services/review-classifier/acp-provider`             | you, in the store's UI, field `api_key`                                              | `swarm-review-classifier`                           | ❌ write a new key                                                                                             | ✅ read on every run                                                                                                                                                                                                                                         |
-| `swarm/services/review-classifier/forge/token`              | `swarm-controller`, at start and every 5 minutes                                     | `swarm-controller`                                  | ✅ when missing or no longer matching the forge                                                                | ✅ read on every review                                                                                                                                                                                                                                      |
-| `swarm/services/builder/forge/token`                        | `swarm-controller`, at start and every 5 minutes                                     | `swarm-builder`, `swarm-controller`                 | ✅ when missing or no longer matching the forge                                                                | ✅ read on every build                                                                                                                                                                                                                                       |
-| `swarm/services/ci-metrics/forge/token`                     | `swarm-controller`, at start and every 5 minutes                                     | `swarm-ci-metrics`, `swarm-controller`              | ✅ when missing or no longer matching the forge                                                                | ✅ read on every pass                                                                                                                                                                                                                                        |
-| _(not in the store)_ a hive's mTLS leaf                     | the store's PKI, or you by hand                                                      | the hive itself, from disk                          | must be stated                                                                                                 | must be stated                                                                                                                                                                                                                                               |
+| store path                                                  | minter                                                                                                                         | reader                                              | automatic re-mint                                                                                              | automatic re-pull                                                                                                                                                                                                                                            |
+| ----------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `swarm/agents/<agent>/matrix/main`                          | `swarm-controller`, at agent creation and every 5 minutes                                                                      | the agent                                           | ✅ when missing or rejected by the homeserver                                                                  | ✅ the daemon restarts within 5 minutes of a rejection                                                                                                                                                                                                       |
+| `swarm/agents/<agent>/matrix/<account>`                     | `swarm-controller`, from a credential you hand it                                                                              | the agent                                           | must be stated                                                                                                 | ✅ retried within 2 to 16 minutes after a rejection                                                                                                                                                                                                          |
+| `swarm/agents/<agent>/matrix-recovery/<account>`            | `swarm-controller`, a random passphrase per account                                                                            | the agent                                           | ❌ never replaced: the account's key backup opens only with this passphrase                                    | ❌ read when the account comes up                                                                                                                                                                                                                            |
+| `swarm/controller/swarm-controller/matrix/appservice-token` | `swarm-matrix-ctl` in `hive-matrix`, once                                                                                      | `swarm-controller`                                  | ❌ minted once                                                                                                 | ✅ read every 5 minutes                                                                                                                                                                                                                                      |
+| `swarm/controller/swarm-controller/oidc/client`             | authelia, first boot                                                                                                           | `swarm-controller`                                  | ❌ minted once; `swarm-secret-publish` republishes a re-mint                                                   | ✅ re-read when authelia refuses it                                                                                                                                                                                                                          |
+| `swarm/controller/swarm-controller/forge/token`             | the forge, once                                                                                                                | `swarm-controller`                                  | ❌ minted once; delete `/var/lib/forgejo/swarm-controller-token-name` in the forge container to mint a new one | ✅ re-read on a 401                                                                                                                                                                                                                                          |
+| `swarm/controller/swarm-controller/grafana/token`           | `swarm-bao-grafana-controller-token` on Grafana's host                                                                         | `swarm-controller`                                  | ✅ every 10 minutes, when Grafana refuses it or none is stored                                                 | ✅ re-read on a 401                                                                                                                                                                                                                                          |
+| `swarm/controller/swarm-controller/forge/webhook-secret`    | `swarm-controller`, at start, from its old `webhook-secret` file; a new one only when there is no file                         | `swarm-controller`                                  | ❌ minted once; a rotated secret fails every delivery until the forge hooks are re-registered                  | ❌ read at start                                                                                                                                                                                                                                             |
+| `swarm/agents/<agent>/bao-mtls`                             | the store's agent PKI mount (`deploy.bao.agentPkiMountPath`), at agent creation                                                | `hive-c0re`, then the agent                         | ✅ re-issued every 5 minutes once past half its 7-day validity                                                 | ✅ the agent re-reads every 10 minutes; it renews only while its current leaf is valid                                                                                                                                                                       |
+| `swarm/agents/<agent>/queue`                                | `swarm-controller`, at agent creation                                                                                          | the agent; the queue, log and metrics auth services | ✅ re-minted at 45 days; skipped for `Destroyed` agents, whose path is deleted                                 | ✅ read on every reconnect; an open connection keeps the old secret for up to an hour; the log and metrics stores honour a replaced secret for up to `deploy.victorialogs.authVerdictTtl` and `deploy.victoriametrics.agentAuthVerdictTtl` (60 s by default) |
+| `swarm/agents/<agent>/forge-token`                          | `swarm-controller`, at agent creation and every 5 minutes                                                                      | the agent                                           | ✅ when missing or no longer matching the forge                                                                | ✅ every 10 minutes                                                                                                                                                                                                                                          |
+| `swarm/agents/<agent>/forge/<label>`                        | `swarm-controller`, when you link an external forge account in the swarm UI                                                    | the agent                                           | ❌ your token; delete it and link the label again                                                              | ✅ every 2 minutes                                                                                                                                                                                                                                           |
+| `swarm/agents/<agent>/github-token`                         | `swarm-controller`, when you link a GitHub account in the swarm UI                                                             | the agent                                           | ❌ your token; delete it and link again                                                                        | ✅ every 2 minutes                                                                                                                                                                                                                                           |
+| `swarm/agents/<agent>/claude-credentials`                   | `swarm-claude-login`, when you log the agent in from its card; one version per login                                           | the agent, while it waits for a login               | ❌ log in again                                                                                                | ✅ every 30 seconds while waiting                                                                                                                                                                                                                            |
+| `swarm/hives/<hive>/matrix/sender-token`                    | `swarm-controller`, every 5 minutes                                                                                            | `swarm-controller` and the hive                     | ✅ when missing or rejected by the homeserver                                                                  | ✅ the hive re-reads on every sweep                                                                                                                                                                                                                          |
+| `swarm/hives/<hive>/forge/token`                            | `swarm-controller`, every 5 minutes                                                                                            | `swarm-controller` and the hive                     | ✅ when missing or no longer matching the forge                                                                | ✅ read on every forge call                                                                                                                                                                                                                                  |
+| `swarm/hives/<hive>/oidc/client`                            | authelia, first boot; `swarm-secret-publish` copies it in                                                                      | the hive                                            | ❌ minted once; `swarm-secret-publish` republishes a re-mint                                                   | ✅ re-read when authelia refuses it                                                                                                                                                                                                                          |
+| `swarm/services/<clientId>/oidc/client`                     | authelia                                                                                                                       | that service                                        | must be stated                                                                                                 | must be stated                                                                                                                                                                                                                                               |
+| `swarm/services/ci-runner/registration-token`               | `swarm-controller`, from the forge, at start and every 5 minutes                                                               | `swarm-bao-ci-runner` on the runner's host          | ❌ the forge keeps one token until an admin resets it; the controller then writes the new one within 5 minutes | ✅ every 10 minutes; restarts the runner on a change                                                                                                                                                                                                         |
+| `swarm/services/<grafana client id>/metrics/bearer`         | `swarm-bao-grafana-metrics-bearer` on Grafana's host, when none is stored                                                      | `swarm-metrics-auth` and `swarm-logs-grafana-auth`  | ❌ delete it and restart `swarm-bao-grafana-metrics-bearer`                                                    | ✅ the auth services re-read within a minute; ❌ Grafana restarts when its copy changes                                                                                                                                                                      |
+| `swarm/services/<grafana client id>/secret-key`             | `swarm-bao-grafana-secret-key` on Grafana's host, from the key an earlier release generated; a new one only when there is none | Grafana                                             | ❌ minted once; a different key leaves Grafana unable to read its stored datasource secrets                    | ❌ Grafana reads it at start; the unit restarts Grafana when the file changes                                                                                                                                                                                |
+| `swarm/services/review-classifier/anthropic-api-key`        | you, in the store's UI, field `api_key`                                                                                        | `swarm-review-classifier`                           | ❌ write a new key                                                                                             | ✅ read on every run                                                                                                                                                                                                                                         |
+| `swarm/services/review-classifier/acp-provider`             | you, in the store's UI, field `api_key`                                                                                        | `swarm-review-classifier`                           | ❌ write a new key                                                                                             | ✅ read on every run                                                                                                                                                                                                                                         |
+| `swarm/services/review-classifier/forge/token`              | `swarm-controller`, at start and every 5 minutes                                                                               | `swarm-controller`                                  | ✅ when missing or no longer matching the forge                                                                | ✅ read on every review                                                                                                                                                                                                                                      |
+| `swarm/services/builder/forge/token`                        | `swarm-controller`, at start and every 5 minutes                                                                               | `swarm-builder`, `swarm-controller`                 | ✅ when missing or no longer matching the forge                                                                | ✅ read on every build                                                                                                                                                                                                                                       |
+| `swarm/services/ci-metrics/forge/token`                     | `swarm-controller`, at start and every 5 minutes                                                                               | `swarm-ci-metrics`, `swarm-controller`              | ✅ when missing or no longer matching the forge                                                                | ✅ read on every pass                                                                                                                                                                                                                                        |
+| _(not in the store)_ a hive's mTLS leaf                     | the store's PKI, or you by hand                                                                                                | the hive itself, from disk                          | must be stated                                                                                                 | must be stated                                                                                                                                                                                                                                               |
 
 <!-- vale write-good.Passive = YES -->
 
@@ -104,6 +106,8 @@ pieces fit, and the reasoning behind them, is in [`credentials.md`](credentials.
 | Grafana's copy of its OIDC secret                                                                    | `swarm-bao-grafana-oidc.service` reads it out of the swarm secret store, **on every host that runs Grafana**                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   | `/run/swarm-grafana-secrets/<id>.secret` on the host's tmpfs, bind-mounted read-only into the `swarm-grafana` container — never on persistent disk, so a reboot fetches it again                                                                                                                                                                                                                                                         | same unit, same path — one route, co-located or not. Nothing for an operator to place beyond this host's store leaf, see [hive-level secrets](credentials.md#hive-level-secrets)                                                                                                                                                                                                                                                                                                                          |
 | Grafana's admin password                                                                             | `swarm-bao-grafana-admin.service` creates it in the swarm secret store at `swarm/services/<grafana client id>/admin/password` when the store holds none, then copies it out, **on every host that runs Grafana**. Grafana's leaf may create that leaf but not update it, so the store keeps the first value, and the secret publisher can't write it. To rotate, run `bao kv metadata delete secret/swarm/services/<grafana client id>/admin/password` with a token that may, such as the root token (Grafana's, the publisher's and the controller's can't), then `systemctl restart swarm-bao-grafana-admin`; a plain `bao kv delete` isn't enough, see [the callout xkey's section](credentials.md#the-queues-callout-xkey) | `/run/swarm-grafana-secrets/admin-password` on the host's tmpfs, bind-mounted read-only into the `swarm-grafana` container, never on persistent disk: `security.admin_password` for a new database, and `grafana cli admin reset-admin-password` in `grafana.service`'s `preStart` for an existing one, on every start                                                                                                                   | same unit, same path. After a reboot the unit fetches the file again, and Grafana doesn't start until it exists, so it stays down rather than falling back to the upstream default `admin`                                                                                                                                                                                                                                                                                                                |
 | Grafana's metrics bearer                                                                             | `swarm-bao-grafana-metrics-bearer.service` creates it in the swarm secret store at `swarm/services/<grafana client id>/metrics/bearer` when the store holds none, then copies it out, **on every host that runs Grafana**. Same grant as the admin password, create without update. To rotate, run `bao kv metadata delete secret/swarm/services/<grafana client id>/metrics/bearer` with a token that may, then `systemctl restart swarm-bao-grafana-metrics-bearer`                                                                                                                                                                                                                                                          | `/run/swarm-grafana-secrets/metrics-bearer` on the host's tmpfs, bind-mounted read-only into the `swarm-grafana` container, holding `Bearer <value>`: the metrics datasource's `Authorization` header. The logs datasource sends the same header. `swarm-metrics-auth` on the metrics store's host, and a second instance on the log store's host (`swarm-logs-grafana-auth`), read the stored value themselves, each under its own leaf | same unit, same path. The unit restarts Grafana when the file changes                                                                                                                                                                                                                                                                                                                                                                                                                                     |
+| Grafana's datasource encryption key                                                                  | `swarm-bao-grafana-secret-key.service` stores it in the swarm secret store at `swarm/services/<grafana client id>/secret-key` when the store holds none, then copies it out, **on every host that runs Grafana**. It stores the key an earlier release generated at `/var/lib/grafana-secret/secret_key` in the container, so the key doesn't change, and mints a new one only on a host with no such file. Same grant as the admin password, create without update. The earlier file stays on disk. While the store's policy doesn't grant the leaf yet, or the store is down, the unit runs Grafana on that file                                                                                                             | `/run/swarm-grafana-secrets/secret-key` on the host's tmpfs, bind-mounted read-only into the `swarm-grafana` container: `security.secret_key`                                                                                                                                                                                                                                                                                            | same unit, same path. Grafana doesn't start until the file exists. Changing the key leaves Grafana unable to read the datasource secrets it stored, so never delete the store path to rotate it                                                                                                                                                                                                                                                                                                           |
+| the swarm controller's webhook secret                                                                | `swarm-controller` stores it in the swarm secret store at `swarm/controller/swarm-controller/forge/webhook-secret` when the store holds none: the value of `webhook-secret` in its state directory, or a new one when there is no such file. The state directory file stays on disk and is the fallback while the store is unreachable or its policy doesn't grant the path yet                                                                                                                                                                                                                                                                                                                                                | the swarm secret store; the controller holds it in memory and registers the same value with the forge's hooks                                                                                                                                                                                                                                                                                                                            | nothing to place: the controller's store leaf already exists. A store that stays unreachable on a start with no state directory file leaves the webhook endpoint answering 503                                                                                                                                                                                                                                                                                                                            |
 | the metrics store responder's own store leaf (cert + key)                                            | `signLeaf` in `glue-bao-tls.nix` (`swarm-bao-pki.service`), on the host that runs the secret store, once, subject `deploy.bao.metricsAuthCommonName`. `swarm-bao-metrics-auth-policy.service` writes the `swarm-metrics-auth` role and its read grants on the bearer leaf and on the metrics store forwarder's client secret, nothing else                                                                                                                                                                                                                                                                                                                                                                                     | `/var/lib/swarm-bao-pki/metrics-auth.pem` and `metrics-auth-key.pem` on the store's host, handed to `swarm-metrics-auth` through `LoadCredential` (paths in argv, never values) and read by path by `swarm-victoriametrics-forwarder-oidc`                                                                                                                                                                                               | **the operator** places a leaf with that subject on the metrics store's host and names it in `deploy.victoriametrics.authBaoClientCertFile` / `authBaoClientKeyFile`; evaluation fails naming both until then. If the store is on another host, deploy that host first: the role must exist before the responder's login succeeds, and until it does the gateway refuses every Grafana query (fail closed)                                                                                                |
 | the secret publisher's store leaf (cert + key)                                                       | `signLeaf` in `glue-bao-tls.nix` (`swarm-bao-pki.service`), on the host that runs the secret store, once, subject `deploy.bao.secretPublisherCommonName`. `swarm-bao-secret-publisher-policy.service` writes the `swarm-secret-publisher` role: create and update under `swarm/hives/*`, `swarm/services/*` and the controller's client, and read on authelia's forwarder's client secret alone                                                                                                                                                                                                                                                                                                                                | `/var/lib/swarm-bao-pki/secret-publisher.pem` and `secret-publisher-key.pem` on the host, handed into authelia's container as nspawn system credentials, which `swarm-secret-publish` and the forwarder's `swarm-journal-forwarder-oidc` load with `LoadCredential`. nspawn won't start authelia's container while a named file is missing                                                                                               | **the operator** copies the leaf to authelia's host, beside the hive's own store leaf, where `glue-secret-publisher-bao-identity.nix` looks, or names it in `deploy.swarm-secret-publisher.baoClientCertFile` / `baoClientKeyFile`                                                                                                                                                                                                                                                                        |
 | the swarm controller's copy of its OIDC secret                                                       | `swarm-controller` reads it out of the swarm secret store at start, under its own leaf                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         | the daemon's memory only; never on disk                                                                                                                                                                                                                                                                                                                                                                                                  | same read, co-located or not. Nothing for an operator to place beyond the controller's store leaf (`deploy.swarm-controller.baoClientCertFile`)                                                                                                                                                                                                                                                                                                                                                           |
@@ -145,18 +149,18 @@ a role and a policy of its own, and reaches one path, except the review
 classifier's, the CI runner's, the builder's, the CI run poller's, the claude
 login minter's, the forge host's and Grafana's, which reach more than one:
 
-| unit                                                                                                    | option pair under `deploy.bao.`                                    | reads, or writes                                                                                                                                                                             |
-| ------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `swarm-bao-grafana-oidc`, `swarm-bao-grafana-admin`, and Grafana's journal forwarder in its container   | `grafanaOidcClientCertFile` / `grafanaOidcClientKeyFile`           | reads `swarm/services/<grafana client id>/oidc/client` and `swarm/services/<grafana forwarder client id>/oidc/client`; creates and reads `swarm/services/<grafana client id>/admin/password` |
-| `swarm-bao-otel-oidc`                                                                                   | `otelOidcClientCertFile` / `otelOidcClientKeyFile`                 | `swarm/services/<collector client id>/oidc/client`                                                                                                                                           |
-| `swarm-bao-forwarder-oidc`                                                                              | `forwarderOidcClientCertFile` / `forwarderOidcClientKeyFile`       | `swarm/services/<store forwarder client id>/oidc/client`                                                                                                                                     |
-| `swarm-bao-ci-runner`, and the runner's journal forwarder in its container                              | `ciRunnerClientCertFile` / `ciRunnerClientKeyFile`                 | `swarm/services/ci-runner/registration-token` and `swarm/services/<runner forwarder client id>/oidc/client`                                                                                  |
-| `swarm-review-classifier`, in its container                                                             | `reviewClassifierClientCertFile` / `reviewClassifierClientKeyFile` | `swarm/services/review-classifier/anthropic-api-key`, `swarm/services/review-classifier/acp-provider` and `swarm/services/<classifier client id>/oidc/client`                                |
-| `swarm-claude-login`, in its container                                                                  | `claudeLoginClientCertFile` / `claudeLoginClientKeyFile`           | creates and updates `swarm/agents/+/claude-credentials`; reads `swarm/services/<minter client id>/oidc/client`                                                                               |
-| `swarm-notify-router`, in its container                                                                 | `notifyRouterClientCertFile` / `notifyRouterClientKeyFile`         | `swarm/services/<router client id>/oidc/client`                                                                                                                                              |
-| `swarm-builder`, in its container                                                                       | `builderClientCertFile` / `builderClientKeyFile`                   | `swarm/services/builder/forge/token` and `swarm/services/<builder client id>/oidc/client`                                                                                                    |
-| `swarm-ci-metrics`                                                                                      | `ciMetricsClientCertFile` / `ciMetricsClientKeyFile`               | `swarm/services/ci-metrics/forge/token` and `swarm/services/<poller client id>/oidc/client`                                                                                                  |
-| `swarm-bao-forge-oidc` and the forge's controller-token mint and its journal forwarder in its container | `forgePublisherClientCertFile` / `forgePublisherClientKeyFile`     | writes `swarm/controller/swarm-controller/forge/token`; reads `swarm/services/<forge client id>/oidc/client` and `swarm/services/<forge forwarder client id>/oidc/client`                    |
+| unit                                                                                                                                  | option pair under `deploy.bao.`                                    | reads, or writes                                                                                                                                                                                                                                 |
+| ------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `swarm-bao-grafana-oidc`, `swarm-bao-grafana-admin`, `swarm-bao-grafana-secret-key`, and Grafana's journal forwarder in its container | `grafanaOidcClientCertFile` / `grafanaOidcClientKeyFile`           | reads `swarm/services/<grafana client id>/oidc/client` and `swarm/services/<grafana forwarder client id>/oidc/client`; creates and reads `swarm/services/<grafana client id>/admin/password` and `swarm/services/<grafana client id>/secret-key` |
+| `swarm-bao-otel-oidc`                                                                                                                 | `otelOidcClientCertFile` / `otelOidcClientKeyFile`                 | `swarm/services/<collector client id>/oidc/client`                                                                                                                                                                                               |
+| `swarm-bao-forwarder-oidc`                                                                                                            | `forwarderOidcClientCertFile` / `forwarderOidcClientKeyFile`       | `swarm/services/<store forwarder client id>/oidc/client`                                                                                                                                                                                         |
+| `swarm-bao-ci-runner`, and the runner's journal forwarder in its container                                                            | `ciRunnerClientCertFile` / `ciRunnerClientKeyFile`                 | `swarm/services/ci-runner/registration-token` and `swarm/services/<runner forwarder client id>/oidc/client`                                                                                                                                      |
+| `swarm-review-classifier`, in its container                                                                                           | `reviewClassifierClientCertFile` / `reviewClassifierClientKeyFile` | `swarm/services/review-classifier/anthropic-api-key`, `swarm/services/review-classifier/acp-provider` and `swarm/services/<classifier client id>/oidc/client`                                                                                    |
+| `swarm-claude-login`, in its container                                                                                                | `claudeLoginClientCertFile` / `claudeLoginClientKeyFile`           | creates and updates `swarm/agents/+/claude-credentials`; reads `swarm/services/<minter client id>/oidc/client`                                                                                                                                   |
+| `swarm-notify-router`, in its container                                                                                               | `notifyRouterClientCertFile` / `notifyRouterClientKeyFile`         | `swarm/services/<router client id>/oidc/client`                                                                                                                                                                                                  |
+| `swarm-builder`, in its container                                                                                                     | `builderClientCertFile` / `builderClientKeyFile`                   | `swarm/services/builder/forge/token` and `swarm/services/<builder client id>/oidc/client`                                                                                                                                                        |
+| `swarm-ci-metrics`                                                                                                                    | `ciMetricsClientCertFile` / `ciMetricsClientKeyFile`               | `swarm/services/ci-metrics/forge/token` and `swarm/services/<poller client id>/oidc/client`                                                                                                                                                      |
+| `swarm-bao-forge-oidc` and the forge's controller-token mint and its journal forwarder in its container                               | `forgePublisherClientCertFile` / `forgePublisherClientKeyFile`     | writes `swarm/controller/swarm-controller/forge/token`; reads `swarm/services/<forge client id>/oidc/client` and `swarm/services/<forge forwarder client id>/oidc/client`                                                                        |
 
 The option names are `deploy.bao.<pair>`; the common names, what a missing leaf
 costs and how to place a leaf by hand are in
diff --git a/nix/host-modules/glue-bao-readers-policy-order.nix b/nix/host-modules/glue-bao-readers-policy-order.nix
index d74a6973..5401dcab 100644
--- a/nix/host-modules/glue-bao-readers-policy-order.nix
+++ b/nix/host-modules/glue-bao-readers-policy-order.nix
@@ -61,8 +61,8 @@ let
       ];
     };
     # ./swarm-grafana.nix, under Grafana's leaf, which may create the admin
-    # password, the metrics bearer and the controller's Grafana token only
-    # once that policy is written.
+    # password, the metrics bearer, the datasource key and the controller's
+    # Grafana token only once that policy is written.
     swarm-bao-grafana-admin = {
       here = deployCfg.grafana.enable;
       policies = [ "swarm-bao-grafana-oidc-policy" ];
@@ -71,6 +71,10 @@ let
       here = deployCfg.grafana.enable;
       policies = [ "swarm-bao-grafana-oidc-policy" ];
     };
+    swarm-bao-grafana-secret-key = {
+      here = deployCfg.grafana.enable;
+      policies = [ "swarm-bao-grafana-oidc-policy" ];
+    };
     swarm-bao-grafana-controller-token = {
       here = deployCfg.grafana.enable;
       policies = [ "swarm-bao-grafana-oidc-policy" ];
diff --git a/nix/host-modules/glue-bao-readers-store-order.nix b/nix/host-modules/glue-bao-readers-store-order.nix
index 4fc9e438..b746a2c8 100644
--- a/nix/host-modules/glue-bao-readers-store-order.nix
+++ b/nix/host-modules/glue-bao-readers-store-order.nix
@@ -54,6 +54,7 @@ let
     swarm-bao-grafana-oidc = deployCfg.grafana.enable;
     swarm-bao-grafana-admin = deployCfg.grafana.enable;
     swarm-bao-grafana-metrics-bearer = deployCfg.grafana.enable;
+    swarm-bao-grafana-secret-key = deployCfg.grafana.enable;
     swarm-bao-grafana-controller-token = deployCfg.grafana.enable;
     # ./swarm-nats.nix
     swarm-bao-nats-callout-xkey = deployCfg.nats.enable;
diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix
index 270e78cf..c27b4ac1 100644
--- a/nix/host-modules/swarm-bao.nix
+++ b/nix/host-modules/swarm-bao.nix
@@ -514,6 +514,10 @@ let
       capabilities = ["read"]
     }
 
+    path "${credentialMountPath}/data/${controllerWebhookSecretLeaf}" {
+      capabilities = ["create", "read"]
+    }
+
     path "${agentPkiMountPath}/issue/${agentPkiRoleName}" {
       capabilities = ["update"]
     }
@@ -581,6 +585,10 @@ let
       capabilities = ["deny"]
     }
 
+    path "${credentialMountPath}/data/${grafanaSecretKeyLeaf}" {
+      capabilities = ["deny"]
+    }
+
     path "${credentialMountPath}/data/${classifierForgeTokenLeaf}" {
       capabilities = ["deny"]
     }
@@ -656,6 +664,12 @@ let
   # writes it; the controller reads it and holds no other copy.
   controllerGrafanaTokenLeaf = "swarm/controller/swarm-controller/grafana/token";
 
+  # The HMAC secret of the swarm-wide forge webhooks, the nix half of
+  # `swarm_secret_client::forge::controller_webhook_secret_path`. The controller
+  # creates it, seeded from its old state-directory file, and reads it; `create`
+  # without `update`, so the first value stored is the one every start reads.
+  controllerWebhookSecretLeaf = "swarm/controller/swarm-controller/forge/webhook-secret";
+
   # The forge's runner registration token, the nix half of
   # `swarm_secret_client::ci::registration_token_path`. The controller writes
   # it and cannot read it back (`forge::runner_token` keeps the value it last
@@ -963,7 +977,8 @@ let
   #
   # Grafana's leaf also creates its admin password (`swarm-bao-grafana-admin`
   # in ./swarm-grafana.nix) and the bearer its metrics datasource presents
-  # (`swarm-bao-grafana-metrics-bearer`). `create` without `update`, so a stored
+  # (`swarm-bao-grafana-metrics-bearer`) and its datasource-encryption key
+  # (`swarm-bao-grafana-secret-key`). `create` without `update`, so a stored
   # value is never replaced.
   #
   # The controller's Grafana token is the one it may replace:
@@ -979,6 +994,7 @@ let
         + readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.grafana.otel.clientId}/oidc/client"
         + createStanza "${credentialMountPath}/data/${grafanaAdminLeaf}"
         + createStanza "${credentialMountPath}/data/${grafanaMetricsBearerLeaf}"
+        + createStanza "${credentialMountPath}/data/${grafanaSecretKeyLeaf}"
         + ''
           path "${credentialMountPath}/data/${controllerGrafanaTokenLeaf}" {
             capabilities = ["create", "read", "update"]
@@ -1056,6 +1072,10 @@ let
   # ./swarm-victoriametrics.nix and ./swarm-victorialogs.nix.
   grafanaMetricsBearerLeaf = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer";
 
+  # Grafana's datasource-encryption key, which only Grafana's leaf creates. The
+  # nix half of `secretKeyStorePath` in ./swarm-grafana.nix.
+  grafanaSecretKeyLeaf = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/secret-key";
+
   createStanza = path: ''
     path "${path}" {
       capabilities = ["create", "read"]
diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix
index 260f24c8..a5bbb2f4 100644
--- a/nix/host-modules/swarm-grafana.nix
+++ b/nix/host-modules/swarm-grafana.nix
@@ -358,10 +358,17 @@ let
   # looks.
   storeSecretPath = "secret/swarm/services/${cfg.oidc.clientId}/oidc/client";
 
-  # Grafana's own datasource-encryption key. Generated in-container (see the
-  # unit below) because nothing outside the container ever reads it — unlike
-  # the OIDC secret above, whose other reader is authelia's container.
-  secretKeyPath = "/var/lib/grafana-secret/secret_key";
+  # Grafana's datasource-encryption key. Minted once into the store by
+  # `swarm-bao-grafana-secret-key` below, seeded from the file an earlier
+  # release generated in the container, which `swarm-bao.nix`'s
+  # `grafanaOidcReaders` grants create and read on this one leaf. Rotating it
+  # does not re-encrypt what it already encrypted, so a different key leaves
+  # Grafana unable to read its stored datasource secrets.
+  secretKeyStorePath = "secret/swarm/services/${cfg.oidc.clientId}/secret-key";
+  secretKeyPath = "${secretDir}/secret-key";
+  # The earlier file as the host sees it. Never removed: it is the seed, and
+  # the key Grafana runs on while the store's policy does not grant the leaf.
+  legacySecretKeyPath = "/var/lib/nixos-containers/${cfg.machine}/var/lib/grafana-secret/secret_key";
 
   # The built-in `admin` account's password. Grafana's API takes basic auth
   # whatever the login form says, so this account is reachable from the public
@@ -1019,6 +1026,122 @@ in
       '';
     };
 
+    # Grafana's datasource-encryption key: the store's value is copied into the
+    # container; when the store holds none, the earlier file's key is stored
+    # first, and a new key only when there is no earlier file either. Same
+    # shape and grant as the admin password above.
+    #
+    # The key never changes. An earlier release generated it in the container
+    # on first boot and kept it, so a host that has run Grafana stores that
+    # same key. While the store's policy does not grant the leaf yet (the
+    # store's host is deployed after this one) or the store is unreachable,
+    # the earlier file is what Grafana runs on, and the unit succeeds; a host
+    # without one has no key to keep, so there the unit fails and retries.
+    systemd.services.swarm-bao-grafana-secret-key = {
+      description = "create Grafana's datasource encryption key in the swarm secret store if absent and fetch it";
+      # Ordered after the store where it runs here:
+      # ./glue-bao-readers-store-order.nix.
+      before = [ "container@${cfg.machine}.service" ];
+      wantedBy = [
+        "multi-user.target"
+        "container@${cfg.machine}.service"
+      ];
+      path = [
+        baoDeploy.package
+        pkgs.coreutils
+        pkgs.openssl
+        pkgs.systemd
+      ];
+      inherit (storeRetry) startLimitBurst startLimitIntervalSec;
+      serviceConfig = storeRetry.serviceConfig // {
+        Type = "oneshot";
+        RemainAfterExit = true;
+        SyslogIdentifier = "swarm-bao-grafana-secret-key";
+        TimeoutStartSec = 30;
+      };
+      environment = {
+        BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
+        BAO_CLIENT_CERT = baoDeploy.grafanaOidcClientCertFile;
+        BAO_CLIENT_KEY = baoDeploy.grafanaOidcClientKeyFile;
+      }
+      // lib.optionalAttrs (baoDeploy.serverCaFile != null) {
+        BAO_CACERT = baoDeploy.serverCaFile;
+      };
+      script = ''
+        set -euo pipefail
+
+        ${atomicWriteSecret}
+        ${refreshConsumer}
+
+        ${ensureSecretDir}
+
+        err="$(mktemp)"
+        trap 'rm -f "$err"' EXIT
+
+        # The earlier file's key, or nothing.
+        earlier_key() {
+          if [ -s ${lib.escapeShellArg legacySecretKeyPath} ]; then
+            tr -d '\n' < ${lib.escapeShellArg legacySecretKeyPath}
+          fi
+        }
+
+        # Owner and mode as for the OIDC secret above, for its reasons.
+        # `$__file{}` is read as Grafana starts, so a changed file needs a
+        # restart.
+        land() {
+          if secret_differs ${lib.escapeShellArg secretKeyPath} "$1"; then
+            atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg secretKeyPath} "$1"
+          fi
+          refresh_consumer ${lib.escapeShellArg cfg.machine} grafana.service ${lib.escapeShellArg secretKeyPath}
+        }
+
+        # The store cannot answer: run on the earlier file's key, or fail.
+        use_earlier_key() {
+          local key
+          key="$(earlier_key)"
+          if [ -z "$key" ]; then
+            echo "$1, and there is no earlier key at ${legacySecretKeyPath}; Grafana does not start without one." >&2
+            exit 1
+          fi
+          echo "$1; running Grafana on the key at ${legacySecretKeyPath}." >&2
+          land "$key"
+          exit 0
+        }
+
+        if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
+          cat "$err" >&2
+          use_earlier_key "could not log in to swarm-bao with Grafana's certificate"
+        fi
+        export BAO_TOKEN
+
+        # Read, create on a miss, read again: the admin password's sequence
+        # above, for its reasons, with the earlier key as the first candidate.
+        if ! key="$(bao kv get -field=value ${lib.escapeShellArg secretKeyStorePath} 2>"$err")"; then
+          candidate="$(earlier_key)"
+          if [ -n "$candidate" ]; then
+            echo "storing the key at ${legacySecretKeyPath} in ${secretKeyStorePath}."
+          else
+            candidate="$(openssl rand -hex 32)"
+            echo "no earlier key; minting one in ${secretKeyStorePath}."
+          fi
+          if ! printf '%s' "$candidate" | bao kv put -cas=0 ${lib.escapeShellArg secretKeyStorePath} value=- >/dev/null 2>"$err"; then
+            echo "could not create ${secretKeyStorePath}; reading it back in case another run created it." >&2
+            cat "$err" >&2
+          fi
+          if ! key="$(bao kv get -field=value ${lib.escapeShellArg secretKeyStorePath} 2>"$err")"; then
+            cat "$err" >&2
+            use_earlier_key "swarm-bao did not return ${secretKeyStorePath}"
+          fi
+        fi
+        if [ -z "$key" ]; then
+          echo "swarm-bao returned an empty ${secretKeyStorePath}; leaving the file as it is." >&2
+          exit 1
+        fi
+
+        land "$key"
+      '';
+    };
+
     # swarm-controller reads Grafana's alerts with a token of a Viewer service
     # account, created here through Grafana's API as the built-in admin and
     # handed over through the store.
@@ -1253,55 +1376,6 @@ in
           };
           systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames storeFiles;
 
-          # Grafana's `secret_key` has **no default in nixpkgs** and an
-          # assertion refuses the build without one — which is how the first
-          # version of this module broke a deploy. It signs the datasource
-          # secrets in Grafana's own database.
-          #
-          # Generated in-container on first boot, like authelia's own keys and
-          # for the same reason: nothing outside this container ever reads it,
-          # which makes in-container generation right rather than merely
-          # easier. (The OIDC client secret is delivered host-side precisely
-          # because it has a second reader.)
-          #
-          # ⚠️ Generated ONCE and kept — the `-s` guard is load-bearing.
-          # Rotating this key does not re-encrypt what it already encrypted, so
-          # a fresh key on every boot would leave Grafana unable to decrypt its
-          # own stored datasource secrets. Under /var/lib, never /run.
-          systemd.services.swarm-grafana-secret-key = {
-            description = "generate Grafana's datasource encryption key on first boot";
-            wantedBy = [ "grafana.service" ];
-            before = [ "grafana.service" ];
-            serviceConfig = {
-              Type = "oneshot";
-              RemainAfterExit = true;
-              # Same User/Group/StateDirectory as grafana itself, so systemd
-              # creates the directory owned by the account that has to read
-              # the file — no chown, no mode juggling.
-              User = "grafana";
-              Group = "grafana";
-              StateDirectory = "grafana-secret";
-              StateDirectoryMode = "0700";
-              SyslogIdentifier = "swarm-grafana-secret-key";
-            };
-            path = [
-              pkgs.openssl
-              pkgs.coreutils
-            ];
-            script = ''
-              set -euo pipefail
-
-              ${atomicWriteSecret}
-
-              key=${lib.escapeShellArg secretKeyPath}
-              if [ ! -s "$key" ]; then
-                value=$(openssl rand -hex 32)
-                atomic_write_secret 0600 "" "$key" "$value"
-                echo "generated Grafana's datasource encryption key"
-              fi
-            '';
-          };
-
           # Upstream applies `security.admin_password` only when it creates the
           # database, so a database created before that setting keeps the
           # password it was created with. This resets it from the delivered
@@ -1418,7 +1492,8 @@ in
 
               # ⚠️ `$__file{}`, and required rather than optional: nixpkgs
               # dropped the default and asserts on null, so without this the
-              # whole host fails to build. The unit below generates it.
+              # whole host fails to build. Written by `swarm-bao-grafana-secret-key`
+              # on the host.
               security.secret_key = "$__file{${secretKeyPath}}";
 
               # The password a newly created database gets; the `preStart`
diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix
index 9cc2fd8d..1c3ad40b 100644
--- a/nix/module-eval/bao-grants.nix
+++ b/nix/module-eval/bao-grants.nix
@@ -483,6 +483,23 @@ let
             "update"
           ];
     }
+    {
+      # Grafana's datasource key sits under the publisher's `services/*` write
+      # too, and a planted value is the key Grafana would encrypt with from its
+      # next start on, orphaning what it stored under the old one. The OIDC
+      # leaf beside it is the control: still the prefix's.
+      name = "the publisher may not write Grafana's datasource key";
+      ok =
+        let
+          grants = grantsIn baoGrantHere.systemd.services.swarm-bao-secret-publisher-policy.script;
+        in
+        (grantFor grants "secret/data/swarm/services/swarm-grafana/secret-key").caps == [ "deny" ]
+        &&
+          (grantFor grants "secret/data/swarm/services/swarm-grafana/oidc/client").caps == [
+            "create"
+            "update"
+          ];
+    }
     {
       # The review classifier's forge token sits under the publisher's
       # `services/*` write too, and the controller posts the classifier's
@@ -611,12 +628,12 @@ let
       # both, and the two are separate principals precisely because a
       # dashboard is not entitled to a collector's credential.
       #
-      # The admin password and the metrics bearer are the leaves it may create,
+      # The admin password, the metrics bearer and the datasource key are the leaves it may create,
       # and without `update`: a grant that could replace one would let a second
       # run rotate a value Grafana or the metrics store already holds. The
       # controller's Grafana token is the one leaf it may replace, which it
       # does when Grafana refuses the stored token.
-      name = "the Grafana reader reads its own and its forwarder's client secrets, creates its admin password and metrics bearer, and writes the controller's Grafana token, and nothing else";
+      name = "the Grafana reader reads its own and its forwarder's client secrets, creates its admin password, metrics bearer and datasource key, and writes the controller's Grafana token, and nothing else";
       ok =
         let
           s = baoGrantHere.systemd.services.swarm-bao-grafana-oidc-policy.script;
@@ -644,6 +661,13 @@ let
               "read"
             ];
           }
+          {
+            path = "secret/data/swarm/services/swarm-grafana/secret-key";
+            caps = [
+              "create"
+              "read"
+            ];
+          }
           {
             path = "secret/data/swarm/controller/swarm-controller/grafana/token";
             caps = [
@@ -2028,6 +2052,24 @@ let
             "swarm-bao-grafana-oidc-policy"
           ];
     }
+    {
+      # The webhook HMAC secret: the controller creates it, seeded from its
+      # old state-directory file, and reads it. `create` without `update`, so
+      # the first value stored is the one every start reads. Pinned as the
+      # whole stanza, so an added capability fails; the sibling token's
+      # stanza above is the presence control for the same matcher.
+      name = "the controller creates and reads its webhook secret, and no other granting unit names it";
+      ok =
+        let
+          s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
+          naming = lib.filter (
+            u: lib.hasInfix "swarm-controller/forge/webhook-secret" baoGrantHere.systemd.services.${u}.script
+          ) grantingUnitNames;
+        in
+        lib.hasInfix "path \"secret/data/swarm/controller/swarm-controller/forge/webhook-secret\" {\n  capabilities = [\"create\", \"read\"]\n}" s
+        && lib.hasInfix "path \"secret/data/swarm/controller/swarm-controller/forge/token\" {\n  capabilities = [\"read\"]\n}" s
+        && naming == [ "swarm-bao-controller-policy" ];
+    }
     {
       # The forge's OIDC client secret by name: the forge publisher reads it,
       # and the secret publisher writes it under its `swarm/services/*` grant,
diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix
index d732fa94..9661aa0e 100644
--- a/nix/module-eval/grafana.nix
+++ b/nix/module-eval/grafana.nix
@@ -250,6 +250,37 @@ let
         && lib.hasInfix "admin reset-admin-password --password-from-stdin < /run/swarm-grafana-secrets/admin-password" service.preStart
         && lib.hasInfix "-config ${resetConfig}" service.script;
     }
+    {
+      # Grafana's datasource-encryption key reaches Grafana from the store,
+      # seeded from the file an earlier release generated in the container,
+      # which stays on disk. A different key leaves Grafana unable to read its
+      # stored datasource secrets, so nothing may mint ahead of that seed, and
+      # the superseded in-container generator must be gone: left in place it
+      # would write a second key beside the delivered one. The last two arms
+      # are the presence controls for the absence arms: the guest's `grafana`
+      # unit is found by the same lookup, and the admin password's legacy file
+      # is among the removals.
+      name = "grafana's datasource key is seeded into the store from the earlier file, delivered from /run, and the earlier file is kept";
+      ok =
+        let
+          m = grafanaRemoteAuthelia;
+          unit = m.systemd.services.swarm-bao-grafana-secret-key;
+          guest = m.containers.swarm-grafana.config;
+          earlier = "/var/lib/nixos-containers/swarm-grafana/var/lib/grafana-secret/secret_key";
+        in
+        lib.hasInfix "bao kv put -cas=0 secret/swarm/services/swarm-grafana/secret-key value=-" unit.script
+        && lib.hasInfix "tr -d '\\n' < ${earlier}" unit.script
+        && lib.hasInfix "atomic_write_secret 0400 196:0 /run/swarm-grafana-secrets/secret-key" unit.script
+        && lib.elem "container@swarm-grafana.service" unit.before
+        && unit.environment.BAO_CLIENT_CERT == "/etc/pki/bao-grafana-oidc.pem"
+        &&
+          guest.services.grafana.settings.security.secret_key
+          == "$__file{/run/swarm-grafana-secrets/secret-key}"
+        && !(guest.systemd.services ? swarm-grafana-secret-key)
+        && guest.systemd.services ? grafana
+        && !(lib.any (lib.hasInfix "grafana-secret") m.systemd.tmpfiles.rules)
+        && lib.elem "r /var/lib/nixos-containers/swarm-grafana/var/lib/grafana-admin/password" m.systemd.tmpfiles.rules;
+    }
     {
       # swarm-controller reads alerts with this token, so the account it
       # belongs to is created and kept a Viewer, the token lands at the path
diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix
index ecb3d04e..b22e038a 100644
--- a/nix/module-eval/swarm-services-switch.nix
+++ b/nix/module-eval/swarm-services-switch.nix
@@ -1214,6 +1214,7 @@ let
             "swarm-bao-grafana-oidc"
             "swarm-bao-grafana-admin"
             "swarm-bao-grafana-metrics-bearer"
+            "swarm-bao-grafana-secret-key"
             "swarm-bao-grafana-controller-token"
             "swarm-bao-nats-callout-xkey"
             "swarm-bao-ci-runner"
@@ -1239,9 +1240,9 @@ let
             !(lib.any (n: lib.elem n (s.after ++ s.wants ++ s.requires)) storeUnits);
         in
         lib.all waits readers
-        && lib.length readersHere == 9
+        && lib.length readersHere == 10
         && lib.elem "swarm-bao-grafana-admin" readersHere
-        && lib.length readersElsewhere == 7
+        && lib.length readersElsewhere == 8
         && lib.all namesNoStoreUnit readersElsewhere
         && !(lib.any (u: storeOnlyNoHive.systemd.services ? ${u}) readers);
     }
diff --git a/swarm-controller/README.md b/swarm-controller/README.md
index 39b363d5..b04adee5 100644
--- a/swarm-controller/README.md
+++ b/swarm-controller/README.md
@@ -115,8 +115,8 @@ The swarm's control plane. The swarm UI and `swarmctl` are its clients.
   → [`docs/swarm/credentials-reference.md`](../docs/swarm/credentials-reference.md).
 
 It reads its configuration once at startup, from the environment the nix module
-sets. It persists `webhook-secret` and `operator-matrix-admins.json` in its
-state directory.
+sets. It keeps the webhook HMAC secret in the secret store and
+`operator-matrix-admins.json` in its state directory.
 The job graph lives in memory; hive status and wanted state live in the swarm
 queue, so both survive a restart.
 
diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs
index 49817605..9f2e5848 100644
--- a/swarm-controller/src/main.rs
+++ b/swarm-controller/src/main.rs
@@ -8,10 +8,10 @@
 //! `startup::load_hives`. A config change means a redeploy, same as every other
 //! option this process reads.
 //!
-//! It persists `webhook-secret` under its `StateDirectory` (see `webhook`),
+//! It keeps the webhook HMAC secret in the swarm secret store (see `webhook`),
 //! because that key is handed to Forgejo at registration and so cannot be
-//! regenerated per boot, and the operator admin sync's granted set beside it
-//! (see `matrix_account::operator_admin`).
+//! regenerated per boot, and the operator admin sync's granted set under its
+//! `StateDirectory` (see `matrix_account::operator_admin`).
 //!
 //! Distinct from `hive-c0re`, which is per-hive: c0re owns the agents on
 //! one host, this owns what is true across hives.
@@ -179,7 +179,7 @@ async fn main() -> Result<()> {
     register_cert_gauges(jobq_metrics_provider.as_ref());
     held_deploys::register(jobq_metrics_provider.as_ref(), &jobq);
 
-    let webhook_secret = load_webhook_secret();
+    let webhook_secret = load_webhook_secret().await;
 
     // Diagnostic only — `put_matrix_account` does not call this yet (see its
     // own doc comment), so this cannot change any request's outcome. Logged
diff --git a/swarm-controller/src/startup.rs b/swarm-controller/src/startup.rs
index 7fe1dabb..97846a91 100644
--- a/swarm-controller/src/startup.rs
+++ b/swarm-controller/src/startup.rs
@@ -217,8 +217,8 @@ pub(crate) fn agent_login(
 /// route; the webhook endpoint answers 503, which is the honest answer
 /// rather than a silent accept. Lifted out of `main` for
 /// `clippy::too_many_lines`.
-pub(crate) fn load_webhook_secret() -> Option<Arc<str>> {
-    match webhook::load_or_generate_secret() {
+pub(crate) async fn load_webhook_secret() -> Option<Arc<str>> {
+    match webhook::load_or_generate_secret().await {
         Ok(secret) => Some(Arc::from(secret)),
         Err(e) => {
             tracing::warn!(
diff --git a/swarm-controller/src/webhook.rs b/swarm-controller/src/webhook.rs
index 80acb52f..f369d956 100644
--- a/swarm-controller/src/webhook.rs
+++ b/swarm-controller/src/webhook.rs
@@ -27,12 +27,13 @@ use sha2::Sha256;
 
 use crate::routes::AppState;
 
-/// Where the HMAC secret is kept, relative to the daemon's state directory.
+/// The legacy home of the HMAC secret, relative to the daemon's state directory.
 ///
-/// The secret is generated on first start and handed to Forgejo when the
-/// hook is registered, so it must survive a restart — a rotated secret
-/// would make every subsequent delivery fail verification with a correctly
-/// configured hook on both sides.
+/// The secret lives in the swarm secret store; this file is where it lived
+/// before, and is read to seed the store. The secret is handed to Forgejo when
+/// the hook is registered, so it must not change — a rotated secret would make
+/// every subsequent delivery fail verification with a correctly configured
+/// hook on both sides.
 const SECRET_FILE: &str = "webhook-secret";
 
 /// Fallback state directory, used only when `STATE_DIRECTORY` is unset (a
@@ -82,44 +83,185 @@ fn state_dir_from(raw: Option<&str>) -> std::path::PathBuf {
     std::path::PathBuf::from(dir)
 }
 
-/// Load the swarm's webhook HMAC secret, generating and persisting it if the
-/// file is absent or malformed.
+/// Load the swarm's webhook HMAC secret from the store, creating it there if
+/// the store holds none.
 ///
-/// Returns a hex-encoded 32-byte secret (64 hex chars).
+/// Returns a hex-encoded 32-byte secret (64 hex chars). Resolution, first
+/// match wins:
+///
+/// 1. The store's value, whatever the legacy file holds.
+/// 2. The store holds none: the legacy file's value is written to the store and
+///    returned, or a fresh one when there is no legacy file. Nothing rotates,
+///    because hooks already registered with the current secret are never
+///    re-registered and would fail every delivery's HMAC check.
+/// 3. The store is unreachable: the legacy file's value, or an error when
+///    there is none. A secret minted while the store may hold another would
+///    rotate it.
+/// 4. The deployment has no store, or its policy does not cover the path yet
+///    (the store's host is deployed after this one): the legacy file as before,
+///    generated if absent. The next start, under the new policy, seeds the
+///    store from it.
+///
+/// The legacy file stays where it is; this never writes it except in case 4.
 ///
 /// # Errors
 ///
-/// Returns an error, leaving the file untouched, when an existing file cannot
-/// be read: hooks already registered with the stored secret are never
-/// re-registered, so replacing it would fail every delivery's HMAC check.
-/// Also errors when the secret cannot be generated (no `/dev/urandom`) or
-/// cannot be persisted. A malformed existing file is **not** an error: it
-/// is regenerated, because a secret that cannot be parsed is
-/// indistinguishable from one that was never written, and failing startup
-/// over it would strand the daemon with no path forward.
-pub(super) fn load_or_generate_secret() -> Result<String> {
-    load_or_generate_at(&secret_path())
+/// Returns an error, leaving the legacy file untouched, when an existing file
+/// cannot be read, when the store is unreachable and no legacy file exists, and
+/// when the secret cannot be generated or persisted. A malformed legacy file is
+/// **not** an error: it counts as absent, because a secret that cannot be
+/// parsed is indistinguishable from one that was never written, and failing
+/// startup over it would strand the daemon with no path forward.
+pub(super) async fn load_or_generate_secret() -> Result<String> {
+    resolve(&StoreSlot, &secret_path()).await
 }
 
-/// The path-taking half of [`load_or_generate_secret`], split out for the
-/// same reason as [`secret_path_from`]: a test can point it at a scratch
-/// file without mutating a process-global env var.
-fn load_or_generate_at(path: &std::path::Path) -> Result<String> {
+/// What the store holds at the webhook secret's path.
+pub(super) enum Stored {
+    /// The secret.
+    Value(String),
+    /// Nothing is stored there.
+    Absent,
+    /// No store is configured, or its policy for this principal does not cover
+    /// the path.
+    NoGrant,
+}
+
+/// The webhook secret's place in the store. A test stands in for the store
+/// through this trait.
+pub(super) trait SecretSlot {
+    /// What is stored now.
+    fn read(&self) -> impl Future<Output = Result<Stored>> + Send;
+
+    /// Store `value` only if nothing is stored yet.
+    fn create(&self, value: &str) -> impl Future<Output = Result<()>> + Send;
+}
+
+/// The slot at [`swarm_secret_client::forge::controller_webhook_secret_path`],
+/// reached under the controller's own store login.
+struct StoreSlot;
+
+impl StoreSlot {
+    async fn read_once() -> Result<Stored> {
+        let path = swarm_secret_client::forge::controller_webhook_secret_path()?;
+        let store = match crate::store::connect().await {
+            Ok(store) => store,
+            Err(swarm_secret_client::Error::MissingEnv(_)) => return Ok(Stored::NoGrant),
+            Err(e) => return Err(e).context("logging in to the swarm secret store"),
+        };
+        match store
+            .read_optional::<swarm_secret_client::forge::WebhookSecret>(&path)
+            .await
+        {
+            Ok(Some(stored)) if stored.value.trim().is_empty() => {
+                anyhow::bail!("{path} holds an empty value")
+            }
+            Ok(Some(stored)) => Ok(Stored::Value(stored.value.trim().to_owned())),
+            Ok(None) => Ok(Stored::Absent),
+            Err(e) if e.is_denied() => Ok(Stored::NoGrant),
+            Err(e) => Err(e).with_context(|| format!("reading {path}")),
+        }
+    }
+}
+
+impl SecretSlot for StoreSlot {
+    /// Retried through a store that is restarting alongside this daemon. A
+    /// denial is an answer, not an outage, so it is not retried.
+    async fn read(&self) -> Result<Stored> {
+        swarm_secret_client::read_with_startup_retry(
+            "reading the webhook secret from the swarm secret store",
+            Self::read_once,
+        )
+        .await
+    }
+
+    async fn create(&self, value: &str) -> Result<()> {
+        let path = swarm_secret_client::forge::controller_webhook_secret_path()?;
+        let store = crate::store::connect()
+            .await
+            .context("logging in to the swarm secret store")?;
+        store
+            .create(
+                &path,
+                &swarm_secret_client::forge::WebhookSecret {
+                    value: value.to_owned(),
+                },
+            )
+            .await
+            .with_context(|| format!("creating {path}"))
+    }
+}
+
+/// [`load_or_generate_secret`]'s resolution, against any slot and legacy file.
+async fn resolve(slot: &impl SecretSlot, legacy: &std::path::Path) -> Result<String> {
+    let stored = match slot.read().await {
+        Ok(stored) => stored,
+        Err(e) => {
+            return match read_legacy(legacy)? {
+                Some(secret) => {
+                    tracing::warn!(
+                        error = %format!("{e:#}"),
+                        path = %legacy.display(),
+                        "swarm secret store unreachable; using the webhook secret file"
+                    );
+                    Ok(secret)
+                }
+                None => Err(e.context("no webhook secret file to fall back on")),
+            };
+        }
+    };
+    match stored {
+        Stored::Value(secret) => Ok(secret),
+        Stored::Absent => seed(slot, legacy).await,
+        Stored::NoGrant => load_or_generate_at(legacy),
+    }
+}
+
+/// Store the legacy file's secret, or a new one when there is none, and return
+/// what the store then holds.
+async fn seed(slot: &impl SecretSlot, legacy: &std::path::Path) -> Result<String> {
+    let secret = if let Some(secret) = read_legacy(legacy)? {
+        tracing::info!(path = %legacy.display(), "storing the webhook secret file's secret in the store");
+        secret
+    } else {
+        tracing::info!("no webhook secret anywhere; generating one in the store");
+        generate_hex_secret()?
+    };
+    if let Err(e) = slot.create(&secret).await {
+        tracing::warn!(error = %format!("{e:#}"), "creating the webhook secret failed; reading back what the store holds");
+    }
+    match slot.read().await? {
+        Stored::Value(stored) => Ok(stored),
+        Stored::Absent | Stored::NoGrant => {
+            anyhow::bail!("the store holds no webhook secret after creating it")
+        }
+    }
+}
+
+/// The legacy file's secret, or `None` when it is absent or malformed.
+fn read_legacy(path: &std::path::Path) -> Result<Option<String>> {
     match std::fs::read_to_string(path) {
         Ok(raw) => {
             let trimmed = raw.trim().to_owned();
             if trimmed.len() == 64 && trimmed.chars().all(|c| c.is_ascii_hexdigit()) {
-                return Ok(trimmed);
+                return Ok(Some(trimmed));
             }
             tracing::warn!(
                 path = %path.display(),
-                "webhook-secret file malformed (wrong length/chars); regenerating"
+                "webhook-secret file malformed (wrong length/chars); ignoring it"
             );
+            Ok(None)
         }
-        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
-        Err(e) => {
-            return Err(e).with_context(|| format!("read webhook secret {}", path.display()));
-        }
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
+        Err(e) => Err(e).with_context(|| format!("read webhook secret {}", path.display())),
+    }
+}
+
+/// The file-only resolution: the secret in `path`, generated and persisted
+/// when the file is absent or malformed.
+fn load_or_generate_at(path: &std::path::Path) -> Result<String> {
+    if let Some(secret) = read_legacy(path)? {
+        return Ok(secret);
     }
     let secret = generate_hex_secret()?;
     std::fs::create_dir_all(path.parent().unwrap_or(path))
@@ -528,7 +670,10 @@ async fn announce_knowledge_change(state: &AppState) {
 
 #[cfg(test)]
 mod tests {
-    use super::{DeliveryKind, Refusal, load_or_generate_at, secret_path_from, verify};
+    use super::{
+        DeliveryKind, Refusal, SecretSlot, Stored, load_or_generate_at, resolve, secret_path_from,
+        verify,
+    };
     use axum::{body::Bytes, http::HeaderMap, http::StatusCode};
     use hmac::{Hmac, KeyInit as _, Mac as _};
     use sha2::Sha256;
@@ -798,4 +943,168 @@ mod tests {
             }
         }
     }
+
+    /// A store that records what it is asked to do.
+    struct FakeSlot {
+        held: std::sync::Mutex<Option<String>>,
+        /// What `read` answers while it holds nothing.
+        empty: fn() -> anyhow::Result<Stored>,
+        creates: std::sync::Mutex<u32>,
+    }
+
+    impl FakeSlot {
+        fn holding(secret: Option<&str>) -> Self {
+            Self {
+                held: std::sync::Mutex::new(secret.map(str::to_owned)),
+                empty: || Ok(Stored::Absent),
+                creates: std::sync::Mutex::new(0),
+            }
+        }
+
+        fn answering(empty: fn() -> anyhow::Result<Stored>) -> Self {
+            Self {
+                empty,
+                ..Self::holding(None)
+            }
+        }
+
+        fn held(&self) -> Option<String> {
+            self.held.lock().expect("unpoisoned").clone()
+        }
+
+        fn creates(&self) -> u32 {
+            *self.creates.lock().expect("unpoisoned")
+        }
+    }
+
+    impl SecretSlot for FakeSlot {
+        async fn read(&self) -> anyhow::Result<Stored> {
+            match self.held() {
+                Some(secret) => Ok(Stored::Value(secret)),
+                None => (self.empty)(),
+            }
+        }
+
+        async fn create(&self, value: &str) -> anyhow::Result<()> {
+            *self.creates.lock().expect("unpoisoned") += 1;
+            self.held
+                .lock()
+                .expect("unpoisoned")
+                .get_or_insert_with(|| value.to_owned());
+            Ok(())
+        }
+    }
+
+    const ON_DISK: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
+    const IN_STORE: &str = "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210";
+
+    /// A scratch legacy file, holding `contents` when given.
+    fn legacy_file(contents: Option<&str>) -> (tempfile::TempDir, std::path::PathBuf) {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let path = dir.path().join("webhook-secret");
+        if let Some(contents) = contents {
+            std::fs::write(&path, format!("{contents}\n")).expect("seed");
+        }
+        (dir, path)
+    }
+
+    #[tokio::test]
+    async fn an_empty_store_is_seeded_from_the_legacy_file_unchanged() {
+        let (_dir, path) = legacy_file(Some(ON_DISK));
+        let slot = FakeSlot::holding(None);
+
+        let secret = resolve(&slot, &path).await.expect("resolves");
+
+        assert_eq!(secret, ON_DISK, "the effective secret must not change");
+        assert_eq!(slot.held().as_deref(), Some(ON_DISK));
+        assert_eq!(
+            std::fs::read_to_string(&path).expect("file kept"),
+            format!("{ON_DISK}\n")
+        );
+    }
+
+    #[tokio::test]
+    async fn the_store_wins_when_both_hold_a_secret() {
+        let (_dir, path) = legacy_file(Some(ON_DISK));
+        let slot = FakeSlot::holding(Some(IN_STORE));
+
+        let secret = resolve(&slot, &path).await.expect("resolves");
+
+        assert_eq!(secret, IN_STORE);
+        assert_eq!(slot.creates(), 0, "a held secret is never written over");
+    }
+
+    #[tokio::test]
+    async fn neither_holding_one_generates_it_in_the_store_only() {
+        let (_dir, path) = legacy_file(None);
+        let slot = FakeSlot::holding(None);
+
+        let secret = resolve(&slot, &path).await.expect("resolves");
+
+        assert_eq!(secret.len(), 64);
+        assert_eq!(slot.held().as_deref(), Some(secret.as_str()));
+        assert!(!path.exists(), "a new secret is not mirrored to disk");
+    }
+
+    #[tokio::test]
+    async fn a_malformed_legacy_file_does_not_seed_the_store() {
+        let (_dir, path) = legacy_file(Some("not-a-hex-secret"));
+        let slot = FakeSlot::holding(None);
+
+        let secret = resolve(&slot, &path).await.expect("resolves");
+
+        assert_eq!(secret.len(), 64);
+        assert_ne!(secret, "not-a-hex-secret");
+        assert_eq!(slot.held().as_deref(), Some(secret.as_str()));
+    }
+
+    #[tokio::test]
+    async fn without_a_grant_the_legacy_file_is_used_and_never_replaced() {
+        let (_dir, path) = legacy_file(Some(ON_DISK));
+        let slot = FakeSlot::answering(|| Ok(Stored::NoGrant));
+
+        let secret = resolve(&slot, &path).await.expect("resolves");
+
+        assert_eq!(secret, ON_DISK);
+        assert_eq!(slot.creates(), 0);
+    }
+
+    #[tokio::test]
+    async fn without_a_grant_and_a_file_the_old_behaviour_generates_one_on_disk() {
+        let (_dir, path) = legacy_file(None);
+        let slot = FakeSlot::answering(|| Ok(Stored::NoGrant));
+
+        let secret = resolve(&slot, &path).await.expect("resolves");
+
+        assert_eq!(secret.len(), 64);
+        assert_eq!(
+            std::fs::read_to_string(&path).expect("persisted"),
+            format!("{secret}\n"),
+            "the next start, under a granting policy, seeds the store from it"
+        );
+        assert_eq!(slot.creates(), 0);
+    }
+
+    #[tokio::test]
+    async fn an_unreachable_store_uses_the_legacy_file() {
+        let (_dir, path) = legacy_file(Some(ON_DISK));
+        let slot = FakeSlot::answering(|| anyhow::bail!("connection refused"));
+
+        let secret = resolve(&slot, &path).await.expect("resolves");
+
+        assert_eq!(secret, ON_DISK);
+    }
+
+    #[tokio::test]
+    async fn an_unreachable_store_and_no_file_mints_nothing() {
+        let (_dir, path) = legacy_file(None);
+        let slot = FakeSlot::answering(|| anyhow::bail!("connection refused"));
+
+        resolve(&slot, &path)
+            .await
+            .expect_err("no secret may be minted while the store may hold one");
+
+        assert!(!path.exists());
+        assert_eq!(slot.creates(), 0);
+    }
 }
diff --git a/swarm-secret-client/src/forge.rs b/swarm-secret-client/src/forge.rs
index 438a9333..1244e38d 100644
--- a/swarm-secret-client/src/forge.rs
+++ b/swarm-secret-client/src/forge.rs
@@ -16,6 +16,9 @@
 //! `swarm-controller`'s own forge token ([`controller_token_path`]) is the
 //! fourth: the forge's `forgejo-swarm-controller-account`, in its container,
 //! mints it and writes it here, and the controller reads it as it starts.
+//!
+//! The webhook HMAC secret ([`controller_webhook_secret_path`]) is the one
+//! value here the controller mints itself.
 
 use serde::{Deserialize, Serialize};
 
@@ -81,6 +84,30 @@ pub fn controller_token_path() -> Result<String, Error> {
     Ok(format!("{prefix}/forge/token"))
 }
 
+/// The path holding the HMAC secret of the swarm-wide forge webhooks.
+///
+/// `swarm-controller` creates it, seeded from the file it kept before the store
+/// held it, and reads it as it starts; it registers the same value with the
+/// forge. The nix half is `swarm-bao.nix`'s `controllerWebhookSecretLeaf`.
+///
+/// # Errors
+/// Never in practice: the name segment is a constant. The `Result` is
+/// [`principal_prefix`]'s.
+pub fn controller_webhook_secret_path() -> Result<String, Error> {
+    let prefix = principal_prefix(Kind::Controller, crate::matrix::CONTROLLER)?;
+    Ok(format!("{prefix}/forge/webhook-secret"))
+}
+
+/// What [`controller_webhook_secret_path`] holds.
+///
+/// No `Debug` derive: `value` signs every delivery the controller accepts.
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct WebhookSecret {
+    /// The hex secret itself. Named `value` like the other single-value
+    /// objects here, so `bao kv get -field=value` reads it.
+    pub value: String,
+}
+
 /// What [`controller_token_path`] holds. No token name beside the value, unlike
 /// [`Credential`]: the mint writes the token alone.
 ///
@@ -188,6 +215,15 @@ mod tests {
         );
     }
 
+    #[test]
+    fn the_webhook_secret_is_under_the_controller_prefix() {
+        // Spelled out: `swarm-bao.nix` grants exactly this string.
+        assert_eq!(
+            controller_webhook_secret_path().expect("a constant segment is legal"),
+            "swarm/controller/swarm-controller/forge/webhook-secret"
+        );
+    }
+
     #[test]
     fn no_hive_policy_reaches_the_controller_token() {
         let policy = crate::policy::effective_hive("alpha");
diff --git a/swarm-secret-client/src/lib.rs b/swarm-secret-client/src/lib.rs
index 38e27a64..a342318f 100644
--- a/swarm-secret-client/src/lib.rs
+++ b/swarm-secret-client/src/lib.rs
@@ -120,6 +120,20 @@ impl From<vaultrs::error::ClientError> for Error {
     }
 }
 
+impl Error {
+    /// Whether the store answered 403: the principal's policy does not cover
+    /// the path. A policy the store's host has not written yet looks the same
+    /// as one that never will, so a caller that can do without the path
+    /// checks this before treating the failure as an outage.
+    #[must_use]
+    pub fn is_denied(&self) -> bool {
+        matches!(
+            self,
+            Self::Vault(e) if matches!(**e, vaultrs::error::ClientError::APIError { code: 403, .. })
+        )
+    }
+}
+
 /// Install `aws-lc-rs` as the process-wide rustls crypto provider. Call it
 /// first thing in `main` of every binary that dials the queue or reaches the
 /// secret store over TLS.
@@ -138,6 +152,21 @@ pub fn install_crypto_provider() {
 mod tests {
     use super::*;
 
+    #[test]
+    fn only_a_403_is_a_denial() {
+        let api = |code| {
+            Error::from(vaultrs::error::ClientError::APIError {
+                code,
+                errors: Vec::new(),
+            })
+        };
+        assert!(api(403).is_denied());
+        // The controls: a miss and an outage are not denials.
+        assert!(!api(404).is_denied());
+        assert!(!api(503).is_denied());
+        assert!(!Error::MissingEnv("BAO_ADDR").is_denied());
+    }
+
     #[test]
     fn a_tls_client_config_builds_once_the_provider_is_installed() {
         install_crypto_provider();