vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

store the forge webhook secret and grafana's secret key in bao, seeded from the existing files

Browse source
This commit is contained in:
atlas 2026-10-27 12:02:44 +01:00 • committed by mara
commit a4e48a7bca
4 changed files with 697 additions and 142 deletions
Download patch file Download diff file Expand all files Collapse all files

8
nix/host-modules/glue-bao-readers-policy-order.nix
View file

@ -61,8 +61,8 @@ let
];
};
# ./swarm-grafana.nix, under Grafana's leaf, which may create the admin
# password, the metrics bearer and the controller's Grafana token only
# once that policy is written.
# password, the metrics bearer, the datasource key and the controller's
# Grafana token only once that policy is written.
swarm-bao-grafana-admin = {
here = deployCfg.grafana.enable;
policies = [ "swarm-bao-grafana-oidc-policy" ];
@ -71,6 +71,10 @@ let
here = deployCfg.grafana.enable;
policies = [ "swarm-bao-grafana-oidc-policy" ];
};
swarm-bao-grafana-secret-key = {
here = deployCfg.grafana.enable;
policies = [ "swarm-bao-grafana-oidc-policy" ];
};
swarm-bao-grafana-controller-token = {
here = deployCfg.grafana.enable;
policies = [ "swarm-bao-grafana-oidc-policy" ];

1
nix/host-modules/glue-bao-readers-store-order.nix
View file

@ -54,6 +54,7 @@ let
swarm-bao-grafana-oidc = deployCfg.grafana.enable;
swarm-bao-grafana-admin = deployCfg.grafana.enable;
swarm-bao-grafana-metrics-bearer = deployCfg.grafana.enable;
swarm-bao-grafana-secret-key = deployCfg.grafana.enable;
swarm-bao-grafana-controller-token = deployCfg.grafana.enable;
# ./swarm-nats.nix
swarm-bao-nats-callout-xkey = deployCfg.nats.enable;

22
nix/host-modules/swarm-bao.nix
View file

@ -514,6 +514,10 @@ let
capabilities = ["read"]
}
path "${credentialMountPath}/data/${controllerWebhookSecretLeaf}" {
capabilities = ["create", "read"]
}
path "${agentPkiMountPath}/issue/${agentPkiRoleName}" {
capabilities = ["update"]
}
@ -581,6 +585,10 @@ let
capabilities = ["deny"]
}
path "${credentialMountPath}/data/${grafanaSecretKeyLeaf}" {
capabilities = ["deny"]
}
path "${credentialMountPath}/data/${classifierForgeTokenLeaf}" {
capabilities = ["deny"]
}
@ -656,6 +664,12 @@ let
# writes it; the controller reads it and holds no other copy.
controllerGrafanaTokenLeaf = "swarm/controller/swarm-controller/grafana/token";
# The HMAC secret of the swarm-wide forge webhooks, the nix half of
# `swarm_secret_client::forge::controller_webhook_secret_path`. The controller
# creates it, seeded from its old state-directory file, and reads it; `create`
# without `update`, so the first value stored is the one every start reads.
controllerWebhookSecretLeaf = "swarm/controller/swarm-controller/forge/webhook-secret";
# The forge's runner registration token, the nix half of
# `swarm_secret_client::ci::registration_token_path`. The controller writes
# it and cannot read it back (`forge::runner_token` keeps the value it last
@ -963,7 +977,8 @@ let
#
# Grafana's leaf also creates its admin password (`swarm-bao-grafana-admin`
# in ./swarm-grafana.nix) and the bearer its metrics datasource presents
# (`swarm-bao-grafana-metrics-bearer`). `create` without `update`, so a stored
# (`swarm-bao-grafana-metrics-bearer`) and its datasource-encryption key
# (`swarm-bao-grafana-secret-key`). `create` without `update`, so a stored
# value is never replaced.
#
# The controller's Grafana token is the one it may replace:
@ -979,6 +994,7 @@ let
+ readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.grafana.otel.clientId}/oidc/client"
+ createStanza "${credentialMountPath}/data/${grafanaAdminLeaf}"
+ createStanza "${credentialMountPath}/data/${grafanaMetricsBearerLeaf}"
+ createStanza "${credentialMountPath}/data/${grafanaSecretKeyLeaf}"
+ ''
path "${credentialMountPath}/data/${controllerGrafanaTokenLeaf}" {
capabilities = ["create", "read", "update"]
@ -1056,6 +1072,10 @@ let
# ./swarm-victoriametrics.nix and ./swarm-victorialogs.nix.
grafanaMetricsBearerLeaf = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer";
# Grafana's datasource-encryption key, which only Grafana's leaf creates. The
# nix half of `secretKeyStorePath` in ./swarm-grafana.nix.
grafanaSecretKeyLeaf = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/secret-key";
createStanza = path: ''
path "${path}" {
capabilities = ["create", "read"]

183
nix/host-modules/swarm-grafana.nix
View file

@ -358,10 +358,17 @@ let
# looks.
storeSecretPath = "secret/swarm/services/${cfg.oidc.clientId}/oidc/client";
# Grafana's own datasource-encryption key. Generated in-container (see the
# unit below) because nothing outside the container ever reads it — unlike
# the OIDC secret above, whose other reader is authelia's container.
secretKeyPath = "/var/lib/grafana-secret/secret_key";
# Grafana's datasource-encryption key. Minted once into the store by
# `swarm-bao-grafana-secret-key` below, seeded from the file an earlier
# release generated in the container, which `swarm-bao.nix`'s
# `grafanaOidcReaders` grants create and read on this one leaf. Rotating it
# does not re-encrypt what it already encrypted, so a different key leaves
# Grafana unable to read its stored datasource secrets.
secretKeyStorePath = "secret/swarm/services/${cfg.oidc.clientId}/secret-key";
secretKeyPath = "${secretDir}/secret-key";
# The earlier file as the host sees it. Never removed: it is the seed, and
# the key Grafana runs on while the store's policy does not grant the leaf.
legacySecretKeyPath = "/var/lib/nixos-containers/${cfg.machine}/var/lib/grafana-secret/secret_key";
# The built-in `admin` account's password. Grafana's API takes basic auth
# whatever the login form says, so this account is reachable from the public
@ -1019,6 +1026,122 @@ in
'';
};
# Grafana's datasource-encryption key: the store's value is copied into the
# container; when the store holds none, the earlier file's key is stored
# first, and a new key only when there is no earlier file either. Same
# shape and grant as the admin password above.
#
# The key never changes. An earlier release generated it in the container
# on first boot and kept it, so a host that has run Grafana stores that
# same key. While the store's policy does not grant the leaf yet (the
# store's host is deployed after this one) or the store is unreachable,
# the earlier file is what Grafana runs on, and the unit succeeds; a host
# without one has no key to keep, so there the unit fails and retries.
systemd.services.swarm-bao-grafana-secret-key = {
description = "create Grafana's datasource encryption key in the swarm secret store if absent and fetch it";
# Ordered after the store where it runs here:
# ./glue-bao-readers-store-order.nix.
before = [ "container@${cfg.machine}.service" ];
wantedBy = [
"multi-user.target"
"container@${cfg.machine}.service"
];
path = [
baoDeploy.package
pkgs.coreutils
pkgs.openssl
pkgs.systemd
];
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
RemainAfterExit = true;
SyslogIdentifier = "swarm-bao-grafana-secret-key";
TimeoutStartSec = 30;
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = baoDeploy.grafanaOidcClientCertFile;
BAO_CLIENT_KEY = baoDeploy.grafanaOidcClientKeyFile;
}
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
BAO_CACERT = baoDeploy.serverCaFile;
};
script = ''
set -euo pipefail
${atomicWriteSecret}
${refreshConsumer}
${ensureSecretDir}
err="$(mktemp)"
trap 'rm -f "$err"' EXIT
# The earlier file's key, or nothing.
earlier_key() {
if [ -s ${lib.escapeShellArg legacySecretKeyPath} ]; then
tr -d '\n' < ${lib.escapeShellArg legacySecretKeyPath}
fi
}
# Owner and mode as for the OIDC secret above, for its reasons.
# `$__file{}` is read as Grafana starts, so a changed file needs a
# restart.
land() {
if secret_differs ${lib.escapeShellArg secretKeyPath} "$1"; then
atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg secretKeyPath} "$1"
fi
refresh_consumer ${lib.escapeShellArg cfg.machine} grafana.service ${lib.escapeShellArg secretKeyPath}
}
# The store cannot answer: run on the earlier file's key, or fail.
use_earlier_key() {
local key
key="$(earlier_key)"
if [ -z "$key" ]; then
echo "$1, and there is no earlier key at ${legacySecretKeyPath}; Grafana does not start without one." >&2
exit 1
fi
echo "$1; running Grafana on the key at ${legacySecretKeyPath}." >&2
land "$key"
exit 0
}
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
cat "$err" >&2
use_earlier_key "could not log in to swarm-bao with Grafana's certificate"
fi
export BAO_TOKEN
# Read, create on a miss, read again: the admin password's sequence
# above, for its reasons, with the earlier key as the first candidate.
if ! key="$(bao kv get -field=value ${lib.escapeShellArg secretKeyStorePath} 2>"$err")"; then
candidate="$(earlier_key)"
if [ -n "$candidate" ]; then
echo "storing the key at ${legacySecretKeyPath} in ${secretKeyStorePath}."
else
candidate="$(openssl rand -hex 32)"
echo "no earlier key; minting one in ${secretKeyStorePath}."
fi
if ! printf '%s' "$candidate" | bao kv put -cas=0 ${lib.escapeShellArg secretKeyStorePath} value=- >/dev/null 2>"$err"; then
echo "could not create ${secretKeyStorePath}; reading it back in case another run created it." >&2
cat "$err" >&2
fi
if ! key="$(bao kv get -field=value ${lib.escapeShellArg secretKeyStorePath} 2>"$err")"; then
cat "$err" >&2
use_earlier_key "swarm-bao did not return ${secretKeyStorePath}"
fi
fi
if [ -z "$key" ]; then
echo "swarm-bao returned an empty ${secretKeyStorePath}; leaving the file as it is." >&2
exit 1
fi
land "$key"
'';
};
# swarm-controller reads Grafana's alerts with a token of a Viewer service
# account, created here through Grafana's API as the built-in admin and
# handed over through the store.
@ -1253,55 +1376,6 @@ in
};
systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames storeFiles;
# Grafana's `secret_key` has **no default in nixpkgs** and an
# assertion refuses the build without one — which is how the first
# version of this module broke a deploy. It signs the datasource
# secrets in Grafana's own database.
#
# Generated in-container on first boot, like authelia's own keys and
# for the same reason: nothing outside this container ever reads it,
# which makes in-container generation right rather than merely
# easier. (The OIDC client secret is delivered host-side precisely
# because it has a second reader.)
#
# ⚠️ Generated ONCE and kept — the `-s` guard is load-bearing.
# Rotating this key does not re-encrypt what it already encrypted, so
# a fresh key on every boot would leave Grafana unable to decrypt its
# own stored datasource secrets. Under /var/lib, never /run.
systemd.services.swarm-grafana-secret-key = {
description = "generate Grafana's datasource encryption key on first boot";
wantedBy = [ "grafana.service" ];
before = [ "grafana.service" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
# Same User/Group/StateDirectory as grafana itself, so systemd
# creates the directory owned by the account that has to read
# the file — no chown, no mode juggling.
User = "grafana";
Group = "grafana";
StateDirectory = "grafana-secret";
StateDirectoryMode = "0700";
SyslogIdentifier = "swarm-grafana-secret-key";
};
path = [
pkgs.openssl
pkgs.coreutils
];
script = ''
set -euo pipefail
${atomicWriteSecret}
key=${lib.escapeShellArg secretKeyPath}
if [ ! -s "$key" ]; then
value=$(openssl rand -hex 32)
atomic_write_secret 0600 "" "$key" "$value"
echo "generated Grafana's datasource encryption key"
fi
'';
};
# Upstream applies `security.admin_password` only when it creates the
# database, so a database created before that setting keeps the
# password it was created with. This resets it from the delivered
@ -1418,7 +1492,8 @@ in
# ⚠️ `$__file{}`, and required rather than optional: nixpkgs
# dropped the default and asserts on null, so without this the
# whole host fails to build. The unit below generates it.
# whole host fails to build. Written by `swarm-bao-grafana-secret-key`
# on the host.
security.secret_key = "$__file{${secretKeyPath}}";
# The password a newly created database gets; the `preStart`