Watch
0
0
Fork
You've already forked hyperhive
0

store the forge webhook secret and grafana's secret key in bao, seeded from the existing files

This commit is contained in:
atlas 2026-10-27 12:02:44 +01:00 • committed by mara
commit a4e48a7bca
7 changed files with 697 additions and 142 deletions

View file

@ -61,8 +61,8 @@ let
];
};
# ./swarm-grafana.nix, under Grafana's leaf, which may create the admin
# password, the metrics bearer and the controller's Grafana token only
# once that policy is written.
# password, the metrics bearer, the datasource key and the controller's
# Grafana token only once that policy is written.
swarm-bao-grafana-admin = {
here = deployCfg.grafana.enable;
policies = [ "swarm-bao-grafana-oidc-policy" ];
@ -71,6 +71,10 @@ let
here = deployCfg.grafana.enable;
policies = [ "swarm-bao-grafana-oidc-policy" ];
};
swarm-bao-grafana-secret-key = {
here = deployCfg.grafana.enable;
policies = [ "swarm-bao-grafana-oidc-policy" ];
};
swarm-bao-grafana-controller-token = {
here = deployCfg.grafana.enable;
policies = [ "swarm-bao-grafana-oidc-policy" ];

View file

@ -54,6 +54,7 @@ let
swarm-bao-grafana-oidc = deployCfg.grafana.enable;
swarm-bao-grafana-admin = deployCfg.grafana.enable;
swarm-bao-grafana-metrics-bearer = deployCfg.grafana.enable;
swarm-bao-grafana-secret-key = deployCfg.grafana.enable;
swarm-bao-grafana-controller-token = deployCfg.grafana.enable;
# ./swarm-nats.nix
swarm-bao-nats-callout-xkey = deployCfg.nats.enable;

View file

@ -514,6 +514,10 @@ let
capabilities = ["read"]
}
path "${credentialMountPath}/data/${controllerWebhookSecretLeaf}" {
capabilities = ["create", "read"]
}
path "${agentPkiMountPath}/issue/${agentPkiRoleName}" {
capabilities = ["update"]
}
@ -581,6 +585,10 @@ let
capabilities = ["deny"]
}
path "${credentialMountPath}/data/${grafanaSecretKeyLeaf}" {
capabilities = ["deny"]
}
path "${credentialMountPath}/data/${classifierForgeTokenLeaf}" {
capabilities = ["deny"]
}
@ -656,6 +664,12 @@ let
# writes it; the controller reads it and holds no other copy.
controllerGrafanaTokenLeaf = "swarm/controller/swarm-controller/grafana/token";
# The HMAC secret of the swarm-wide forge webhooks, the nix half of
# `swarm_secret_client::forge::controller_webhook_secret_path`. The controller
# creates it, seeded from its old state-directory file, and reads it; `create`
# without `update`, so the first value stored is the one every start reads.
controllerWebhookSecretLeaf = "swarm/controller/swarm-controller/forge/webhook-secret";
# The forge's runner registration token, the nix half of
# `swarm_secret_client::ci::registration_token_path`. The controller writes
# it and cannot read it back (`forge::runner_token` keeps the value it last
@ -963,7 +977,8 @@ let
#
# Grafana's leaf also creates its admin password (`swarm-bao-grafana-admin`
# in ./swarm-grafana.nix) and the bearer its metrics datasource presents
# (`swarm-bao-grafana-metrics-bearer`). `create` without `update`, so a stored
# (`swarm-bao-grafana-metrics-bearer`) and its datasource-encryption key
# (`swarm-bao-grafana-secret-key`). `create` without `update`, so a stored
# value is never replaced.
#
# The controller's Grafana token is the one it may replace:
@ -979,6 +994,7 @@ let
+ readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.grafana.otel.clientId}/oidc/client"
+ createStanza "${credentialMountPath}/data/${grafanaAdminLeaf}"
+ createStanza "${credentialMountPath}/data/${grafanaMetricsBearerLeaf}"
+ createStanza "${credentialMountPath}/data/${grafanaSecretKeyLeaf}"
+ ''
path "${credentialMountPath}/data/${controllerGrafanaTokenLeaf}" {
capabilities = ["create", "read", "update"]
@ -1056,6 +1072,10 @@ let
# ./swarm-victoriametrics.nix and ./swarm-victorialogs.nix.
grafanaMetricsBearerLeaf = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer";
# Grafana's datasource-encryption key, which only Grafana's leaf creates. The
# nix half of `secretKeyStorePath` in ./swarm-grafana.nix.
grafanaSecretKeyLeaf = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/secret-key";
createStanza = path: ''
path "${path}" {
capabilities = ["create", "read"]

View file

@ -358,10 +358,17 @@ let
# looks.
storeSecretPath = "secret/swarm/services/${cfg.oidc.clientId}/oidc/client";
# Grafana's own datasource-encryption key. Generated in-container (see the
# unit below) because nothing outside the container ever reads it — unlike
# the OIDC secret above, whose other reader is authelia's container.
secretKeyPath = "/var/lib/grafana-secret/secret_key";
# Grafana's datasource-encryption key. Minted once into the store by
# `swarm-bao-grafana-secret-key` below, seeded from the file an earlier
# release generated in the container, which `swarm-bao.nix`'s
# `grafanaOidcReaders` grants create and read on this one leaf. Rotating it
# does not re-encrypt what it already encrypted, so a different key leaves
# Grafana unable to read its stored datasource secrets.
secretKeyStorePath = "secret/swarm/services/${cfg.oidc.clientId}/secret-key";
secretKeyPath = "${secretDir}/secret-key";
# The earlier file as the host sees it. Never removed: it is the seed, and
# the key Grafana runs on while the store's policy does not grant the leaf.
legacySecretKeyPath = "/var/lib/nixos-containers/${cfg.machine}/var/lib/grafana-secret/secret_key";
# The built-in `admin` account's password. Grafana's API takes basic auth
# whatever the login form says, so this account is reachable from the public
@ -1019,6 +1026,122 @@ in
'';
};
# Grafana's datasource-encryption key: the store's value is copied into the
# container; when the store holds none, the earlier file's key is stored
# first, and a new key only when there is no earlier file either. Same
# shape and grant as the admin password above.
#
# The key never changes. An earlier release generated it in the container
# on first boot and kept it, so a host that has run Grafana stores that
# same key. While the store's policy does not grant the leaf yet (the
# store's host is deployed after this one) or the store is unreachable,
# the earlier file is what Grafana runs on, and the unit succeeds; a host
# without one has no key to keep, so there the unit fails and retries.
systemd.services.swarm-bao-grafana-secret-key = {
description = "create Grafana's datasource encryption key in the swarm secret store if absent and fetch it";
# Ordered after the store where it runs here:
# ./glue-bao-readers-store-order.nix.
before = [ "container@${cfg.machine}.service" ];
wantedBy = [
"multi-user.target"
"container@${cfg.machine}.service"
];
path = [
baoDeploy.package
pkgs.coreutils
pkgs.openssl
pkgs.systemd
];
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
RemainAfterExit = true;
SyslogIdentifier = "swarm-bao-grafana-secret-key";
TimeoutStartSec = 30;
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = baoDeploy.grafanaOidcClientCertFile;
BAO_CLIENT_KEY = baoDeploy.grafanaOidcClientKeyFile;
}
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
BAO_CACERT = baoDeploy.serverCaFile;
};
script = ''
set -euo pipefail
${atomicWriteSecret}
${refreshConsumer}
${ensureSecretDir}
err="$(mktemp)"
trap 'rm -f "$err"' EXIT
# The earlier file's key, or nothing.
earlier_key() {
if [ -s ${lib.escapeShellArg legacySecretKeyPath} ]; then
tr -d '\n' < ${lib.escapeShellArg legacySecretKeyPath}
fi
}
# Owner and mode as for the OIDC secret above, for its reasons.
# `$__file{}` is read as Grafana starts, so a changed file needs a
# restart.
land() {
if secret_differs ${lib.escapeShellArg secretKeyPath} "$1"; then
atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg secretKeyPath} "$1"
fi
refresh_consumer ${lib.escapeShellArg cfg.machine} grafana.service ${lib.escapeShellArg secretKeyPath}
}
# The store cannot answer: run on the earlier file's key, or fail.
use_earlier_key() {
local key
key="$(earlier_key)"
if [ -z "$key" ]; then
echo "$1, and there is no earlier key at ${legacySecretKeyPath}; Grafana does not start without one." >&2
exit 1
fi
echo "$1; running Grafana on the key at ${legacySecretKeyPath}." >&2
land "$key"
exit 0
}
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
cat "$err" >&2
use_earlier_key "could not log in to swarm-bao with Grafana's certificate"
fi
export BAO_TOKEN
# Read, create on a miss, read again: the admin password's sequence
# above, for its reasons, with the earlier key as the first candidate.
if ! key="$(bao kv get -field=value ${lib.escapeShellArg secretKeyStorePath} 2>"$err")"; then
candidate="$(earlier_key)"
if [ -n "$candidate" ]; then
echo "storing the key at ${legacySecretKeyPath} in ${secretKeyStorePath}."
else
candidate="$(openssl rand -hex 32)"
echo "no earlier key; minting one in ${secretKeyStorePath}."
fi
if ! printf '%s' "$candidate" | bao kv put -cas=0 ${lib.escapeShellArg secretKeyStorePath} value=- >/dev/null 2>"$err"; then
echo "could not create ${secretKeyStorePath}; reading it back in case another run created it." >&2
cat "$err" >&2
fi
if ! key="$(bao kv get -field=value ${lib.escapeShellArg secretKeyStorePath} 2>"$err")"; then
cat "$err" >&2
use_earlier_key "swarm-bao did not return ${secretKeyStorePath}"
fi
fi
if [ -z "$key" ]; then
echo "swarm-bao returned an empty ${secretKeyStorePath}; leaving the file as it is." >&2
exit 1
fi
land "$key"
'';
};
# swarm-controller reads Grafana's alerts with a token of a Viewer service
# account, created here through Grafana's API as the built-in admin and
# handed over through the store.
@ -1253,55 +1376,6 @@ in
};
systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames storeFiles;
# Grafana's `secret_key` has **no default in nixpkgs** and an
# assertion refuses the build without one — which is how the first
# version of this module broke a deploy. It signs the datasource
# secrets in Grafana's own database.
#
# Generated in-container on first boot, like authelia's own keys and
# for the same reason: nothing outside this container ever reads it,
# which makes in-container generation right rather than merely
# easier. (The OIDC client secret is delivered host-side precisely
# because it has a second reader.)
#
# ⚠️ Generated ONCE and kept — the `-s` guard is load-bearing.
# Rotating this key does not re-encrypt what it already encrypted, so
# a fresh key on every boot would leave Grafana unable to decrypt its
# own stored datasource secrets. Under /var/lib, never /run.
systemd.services.swarm-grafana-secret-key = {
description = "generate Grafana's datasource encryption key on first boot";
wantedBy = [ "grafana.service" ];
before = [ "grafana.service" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
# Same User/Group/StateDirectory as grafana itself, so systemd
# creates the directory owned by the account that has to read
# the file — no chown, no mode juggling.
User = "grafana";
Group = "grafana";
StateDirectory = "grafana-secret";
StateDirectoryMode = "0700";
SyslogIdentifier = "swarm-grafana-secret-key";
};
path = [
pkgs.openssl
pkgs.coreutils
];
script = ''
set -euo pipefail
${atomicWriteSecret}
key=${lib.escapeShellArg secretKeyPath}
if [ ! -s "$key" ]; then
value=$(openssl rand -hex 32)
atomic_write_secret 0600 "" "$key" "$value"
echo "generated Grafana's datasource encryption key"
fi
'';
};
# Upstream applies `security.admin_password` only when it creates the
# database, so a database created before that setting keeps the
# password it was created with. This resets it from the delivered
@ -1418,7 +1492,8 @@ in
# ⚠️ `$__file{}`, and required rather than optional: nixpkgs
# dropped the default and asserts on null, so without this the
# whole host fails to build. The unit below generates it.
# whole host fails to build. Written by `swarm-bao-grafana-secret-key`
# on the host.
security.secret_key = "$__file{${secretKeyPath}}";
# The password a newly created database gets; the `preStart`

View file

@ -483,6 +483,23 @@ let
"update"
];
}
{
# Grafana's datasource key sits under the publisher's `services/*` write
# too, and a planted value is the key Grafana would encrypt with from its
# next start on, orphaning what it stored under the old one. The OIDC
# leaf beside it is the control: still the prefix's.
name = "the publisher may not write Grafana's datasource key";
ok =
let
grants = grantsIn baoGrantHere.systemd.services.swarm-bao-secret-publisher-policy.script;
in
(grantFor grants "secret/data/swarm/services/swarm-grafana/secret-key").caps == [ "deny" ]
&&
(grantFor grants "secret/data/swarm/services/swarm-grafana/oidc/client").caps == [
"create"
"update"
];
}
{
# The review classifier's forge token sits under the publisher's
# `services/*` write too, and the controller posts the classifier's
@ -611,12 +628,12 @@ let
# both, and the two are separate principals precisely because a
# dashboard is not entitled to a collector's credential.
#
# The admin password and the metrics bearer are the leaves it may create,
# The admin password, the metrics bearer and the datasource key are the leaves it may create,
# and without `update`: a grant that could replace one would let a second
# run rotate a value Grafana or the metrics store already holds. The
# controller's Grafana token is the one leaf it may replace, which it
# does when Grafana refuses the stored token.
name = "the Grafana reader reads its own and its forwarder's client secrets, creates its admin password and metrics bearer, and writes the controller's Grafana token, and nothing else";
name = "the Grafana reader reads its own and its forwarder's client secrets, creates its admin password, metrics bearer and datasource key, and writes the controller's Grafana token, and nothing else";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-grafana-oidc-policy.script;
@ -644,6 +661,13 @@ let
"read"
];
}
{
path = "secret/data/swarm/services/swarm-grafana/secret-key";
caps = [
"create"
"read"
];
}
{
path = "secret/data/swarm/controller/swarm-controller/grafana/token";
caps = [
@ -2028,6 +2052,24 @@ let
"swarm-bao-grafana-oidc-policy"
];
}
{
# The webhook HMAC secret: the controller creates it, seeded from its
# old state-directory file, and reads it. `create` without `update`, so
# the first value stored is the one every start reads. Pinned as the
# whole stanza, so an added capability fails; the sibling token's
# stanza above is the presence control for the same matcher.
name = "the controller creates and reads its webhook secret, and no other granting unit names it";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
naming = lib.filter (
u: lib.hasInfix "swarm-controller/forge/webhook-secret" baoGrantHere.systemd.services.${u}.script
) grantingUnitNames;
in
lib.hasInfix "path \"secret/data/swarm/controller/swarm-controller/forge/webhook-secret\" {\n capabilities = [\"create\", \"read\"]\n}" s
&& lib.hasInfix "path \"secret/data/swarm/controller/swarm-controller/forge/token\" {\n capabilities = [\"read\"]\n}" s
&& naming == [ "swarm-bao-controller-policy" ];
}
{
# The forge's OIDC client secret by name: the forge publisher reads it,
# and the secret publisher writes it under its `swarm/services/*` grant,

View file

@ -250,6 +250,37 @@ let
&& lib.hasInfix "admin reset-admin-password --password-from-stdin < /run/swarm-grafana-secrets/admin-password" service.preStart
&& lib.hasInfix "-config ${resetConfig}" service.script;
}
{
# Grafana's datasource-encryption key reaches Grafana from the store,
# seeded from the file an earlier release generated in the container,
# which stays on disk. A different key leaves Grafana unable to read its
# stored datasource secrets, so nothing may mint ahead of that seed, and
# the superseded in-container generator must be gone: left in place it
# would write a second key beside the delivered one. The last two arms
# are the presence controls for the absence arms: the guest's `grafana`
# unit is found by the same lookup, and the admin password's legacy file
# is among the removals.
name = "grafana's datasource key is seeded into the store from the earlier file, delivered from /run, and the earlier file is kept";
ok =
let
m = grafanaRemoteAuthelia;
unit = m.systemd.services.swarm-bao-grafana-secret-key;
guest = m.containers.swarm-grafana.config;
earlier = "/var/lib/nixos-containers/swarm-grafana/var/lib/grafana-secret/secret_key";
in
lib.hasInfix "bao kv put -cas=0 secret/swarm/services/swarm-grafana/secret-key value=-" unit.script
&& lib.hasInfix "tr -d '\\n' < ${earlier}" unit.script
&& lib.hasInfix "atomic_write_secret 0400 196:0 /run/swarm-grafana-secrets/secret-key" unit.script
&& lib.elem "container@swarm-grafana.service" unit.before
&& unit.environment.BAO_CLIENT_CERT == "/etc/pki/bao-grafana-oidc.pem"
&&
guest.services.grafana.settings.security.secret_key
== "$__file{/run/swarm-grafana-secrets/secret-key}"
&& !(guest.systemd.services ? swarm-grafana-secret-key)
&& guest.systemd.services ? grafana
&& !(lib.any (lib.hasInfix "grafana-secret") m.systemd.tmpfiles.rules)
&& lib.elem "r /var/lib/nixos-containers/swarm-grafana/var/lib/grafana-admin/password" m.systemd.tmpfiles.rules;
}
{
# swarm-controller reads alerts with this token, so the account it
# belongs to is created and kept a Viewer, the token lands at the path

View file

@ -1214,6 +1214,7 @@ let
"swarm-bao-grafana-oidc"
"swarm-bao-grafana-admin"
"swarm-bao-grafana-metrics-bearer"
"swarm-bao-grafana-secret-key"
"swarm-bao-grafana-controller-token"
"swarm-bao-nats-callout-xkey"
"swarm-bao-ci-runner"
@ -1239,9 +1240,9 @@ let
!(lib.any (n: lib.elem n (s.after ++ s.wants ++ s.requires)) storeUnits);
in
lib.all waits readers
&& lib.length readersHere == 9
&& lib.length readersHere == 10
&& lib.elem "swarm-bao-grafana-admin" readersHere
&& lib.length readersElsewhere == 7
&& lib.length readersElsewhere == 8
&& lib.all namesNoStoreUnit readersElsewhere
&& !(lib.any (u: storeOnlyNoHive.systemd.services ? ${u}) readers);
}