From 859833b18612b02010e8646ff093b74dfbc6bc3b Mon Sep 17 00:00:00 2001
From: atlas <atlas@hyperhive.local>
Date: Sun, 16 Aug 2026 22:38:09 +0200
Subject: [PATCH] fix(#3370): give Grafana a secret_key, generated in-container
 on first boot
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

nixpkgs dropped the default for `services.grafana.settings.security.secret_key`
and asserts on null, so the module I merged an hour ago fails the build on any
host that enables it. That is a broken deploy, not a warning.

Generated in-container like authelia's own keys and for the same reason:
nothing outside the container ever reads it. Generated ONCE and kept — the
key signs Grafana's stored datasource secrets, and rotating it does not
re-encrypt what it already encrypted, so a fresh key per boot would leave
Grafana unable to read its own database. Delivered as `$__file{}`, so it
never enters the store.

The gate missed this because a container is a separate NixOS evaluation with
its own assertions: forcing the host's `config.assertions` never touched the
container's. `state/eval-3265.sh` now forces
`containers.swarm-grafana.config.assertions` and reports any that fail.
---
 nix/host-modules/swarm-grafana.nix | 56 ++++++++++++++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix
index 111bdc03..d5eaf15b 100644
--- a/nix/host-modules/swarm-grafana.nix
+++ b/nix/host-modules/swarm-grafana.nix
@@ -41,6 +41,11 @@ let
   # intermittent one.
   secretPath = "/var/lib/grafana-oidc/${cfg.oidc.clientId}.secret";
 
+  # Grafana's own datasource-encryption key. Generated in-container (see the
+  # unit below) because nothing outside the container ever reads it — unlike
+  # the OIDC secret above, whose other reader is authelia's container.
+  secretKeyPath = "/var/lib/grafana-secret/secret_key";
+
   # Format-locked by Grafana: the generic OAuth callback is always
   # `<root_url>/login/generic_oauth`. Declared once here and read by both
   # the authelia client and Grafana itself.
@@ -344,6 +349,52 @@ in
 
           systemd.services.grafana.environment.SSL_CERT_FILE = lib.mkIf useSelfSigned grafanaCaBundle;
 
+          # Grafana's `secret_key` has **no default in nixpkgs** and an
+          # assertion refuses the build without one — which is how the first
+          # version of this module broke a deploy. It signs the datasource
+          # secrets in Grafana's own database.
+          #
+          # Generated in-container on first boot, like authelia's own keys and
+          # for the same reason: nothing outside this container ever reads it,
+          # which makes in-container generation right rather than merely
+          # easier. (The OIDC client secret is delivered host-side precisely
+          # because it has a second reader.)
+          #
+          # ⚠️ Generated ONCE and kept — the `-s` guard is load-bearing.
+          # Rotating this key does not re-encrypt what it already encrypted, so
+          # a fresh key on every boot would leave Grafana unable to decrypt its
+          # own stored datasource secrets. Under /var/lib, never /run.
+          systemd.services.swarm-grafana-secret-key = {
+            description = "generate Grafana's datasource encryption key on first boot";
+            wantedBy = [ "grafana.service" ];
+            before = [ "grafana.service" ];
+            serviceConfig = {
+              Type = "oneshot";
+              RemainAfterExit = true;
+              # Same User/Group/StateDirectory as grafana itself, so systemd
+              # creates the directory owned by the account that has to read
+              # the file — no chown, no mode juggling.
+              User = "grafana";
+              Group = "grafana";
+              StateDirectory = "grafana-secret";
+              StateDirectoryMode = "0700";
+              SyslogIdentifier = "swarm-grafana-secret-key";
+            };
+            path = [
+              pkgs.openssl
+              pkgs.coreutils
+            ];
+            script = ''
+              set -euo pipefail
+              key=${lib.escapeShellArg secretKeyPath}
+              if [ ! -s "$key" ]; then
+                umask 077
+                openssl rand -hex 32 > "$key"
+                echo "generated Grafana's datasource encryption key"
+              fi
+            '';
+          };
+
           services.grafana = {
             enable = true;
             package = cfg.package;
@@ -371,6 +422,11 @@ in
 
               users.auto_assign_org_role = cfg.oidc.role;
 
+              # ⚠️ `$__file{}`, and required rather than optional: nixpkgs
+              # dropped the default and asserts on null, so without this the
+              # whole host fails to build. The unit below generates it.
+              security.secret_key = "$__file{${secretKeyPath}}";
+
               # No local password path at all when SSO is configured. This
               # is not tidiness: Grafana ships an `admin`/`admin` account,
               # and this vhost is on the public gateway.