store the forge webhook secret and grafana's secret key in bao, seeded from the existing files
This commit is contained in:
atlas
mara
parent
50c1f2a3d1
commit
a4e48a7bca
1 changed files with 697 additions and 142 deletions
|
|
@ -358,10 +358,17 @@ let
|
|||
# looks.
|
||||
storeSecretPath = "secret/swarm/services/${cfg.oidc.clientId}/oidc/client";
|
||||
|
||||
# Grafana's own datasource-encryption key. Generated in-container (see the
|
||||
# unit below) because nothing outside the container ever reads it — unlike
|
||||
# the OIDC secret above, whose other reader is authelia's container.
|
||||
secretKeyPath = "/var/lib/grafana-secret/secret_key";
|
||||
# Grafana's datasource-encryption key. Minted once into the store by
|
||||
# `swarm-bao-grafana-secret-key` below, seeded from the file an earlier
|
||||
# release generated in the container, which `swarm-bao.nix`'s
|
||||
# `grafanaOidcReaders` grants create and read on this one leaf. Rotating it
|
||||
# does not re-encrypt what it already encrypted, so a different key leaves
|
||||
# Grafana unable to read its stored datasource secrets.
|
||||
secretKeyStorePath = "secret/swarm/services/${cfg.oidc.clientId}/secret-key";
|
||||
secretKeyPath = "${secretDir}/secret-key";
|
||||
# The earlier file as the host sees it. Never removed: it is the seed, and
|
||||
# the key Grafana runs on while the store's policy does not grant the leaf.
|
||||
legacySecretKeyPath = "/var/lib/nixos-containers/${cfg.machine}/var/lib/grafana-secret/secret_key";
|
||||
|
||||
# The built-in `admin` account's password. Grafana's API takes basic auth
|
||||
# whatever the login form says, so this account is reachable from the public
|
||||
|
|
@ -1019,6 +1026,122 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
# Grafana's datasource-encryption key: the store's value is copied into the
|
||||
# container; when the store holds none, the earlier file's key is stored
|
||||
# first, and a new key only when there is no earlier file either. Same
|
||||
# shape and grant as the admin password above.
|
||||
#
|
||||
# The key never changes. An earlier release generated it in the container
|
||||
# on first boot and kept it, so a host that has run Grafana stores that
|
||||
# same key. While the store's policy does not grant the leaf yet (the
|
||||
# store's host is deployed after this one) or the store is unreachable,
|
||||
# the earlier file is what Grafana runs on, and the unit succeeds; a host
|
||||
# without one has no key to keep, so there the unit fails and retries.
|
||||
systemd.services.swarm-bao-grafana-secret-key = {
|
||||
description = "create Grafana's datasource encryption key in the swarm secret store if absent and fetch it";
|
||||
# Ordered after the store where it runs here:
|
||||
# ./glue-bao-readers-store-order.nix.
|
||||
before = [ "container@${cfg.machine}.service" ];
|
||||
wantedBy = [
|
||||
"multi-user.target"
|
||||
"container@${cfg.machine}.service"
|
||||
];
|
||||
path = [
|
||||
baoDeploy.package
|
||||
pkgs.coreutils
|
||||
pkgs.openssl
|
||||
pkgs.systemd
|
||||
];
|
||||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||||
serviceConfig = storeRetry.serviceConfig // {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
SyslogIdentifier = "swarm-bao-grafana-secret-key";
|
||||
TimeoutStartSec = 30;
|
||||
};
|
||||
environment = {
|
||||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
BAO_CLIENT_CERT = baoDeploy.grafanaOidcClientCertFile;
|
||||
BAO_CLIENT_KEY = baoDeploy.grafanaOidcClientKeyFile;
|
||||
}
|
||||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||||
BAO_CACERT = baoDeploy.serverCaFile;
|
||||
};
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
${atomicWriteSecret}
|
||||
${refreshConsumer}
|
||||
|
||||
${ensureSecretDir}
|
||||
|
||||
err="$(mktemp)"
|
||||
trap 'rm -f "$err"' EXIT
|
||||
|
||||
# The earlier file's key, or nothing.
|
||||
earlier_key() {
|
||||
if [ -s ${lib.escapeShellArg legacySecretKeyPath} ]; then
|
||||
tr -d '\n' < ${lib.escapeShellArg legacySecretKeyPath}
|
||||
fi
|
||||
}
|
||||
|
||||
# Owner and mode as for the OIDC secret above, for its reasons.
|
||||
# `$__file{}` is read as Grafana starts, so a changed file needs a
|
||||
# restart.
|
||||
land() {
|
||||
if secret_differs ${lib.escapeShellArg secretKeyPath} "$1"; then
|
||||
atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg secretKeyPath} "$1"
|
||||
fi
|
||||
refresh_consumer ${lib.escapeShellArg cfg.machine} grafana.service ${lib.escapeShellArg secretKeyPath}
|
||||
}
|
||||
|
||||
# The store cannot answer: run on the earlier file's key, or fail.
|
||||
use_earlier_key() {
|
||||
local key
|
||||
key="$(earlier_key)"
|
||||
if [ -z "$key" ]; then
|
||||
echo "$1, and there is no earlier key at ${legacySecretKeyPath}; Grafana does not start without one." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "$1; running Grafana on the key at ${legacySecretKeyPath}." >&2
|
||||
land "$key"
|
||||
exit 0
|
||||
}
|
||||
|
||||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||||
cat "$err" >&2
|
||||
use_earlier_key "could not log in to swarm-bao with Grafana's certificate"
|
||||
fi
|
||||
export BAO_TOKEN
|
||||
|
||||
# Read, create on a miss, read again: the admin password's sequence
|
||||
# above, for its reasons, with the earlier key as the first candidate.
|
||||
if ! key="$(bao kv get -field=value ${lib.escapeShellArg secretKeyStorePath} 2>"$err")"; then
|
||||
candidate="$(earlier_key)"
|
||||
if [ -n "$candidate" ]; then
|
||||
echo "storing the key at ${legacySecretKeyPath} in ${secretKeyStorePath}."
|
||||
else
|
||||
candidate="$(openssl rand -hex 32)"
|
||||
echo "no earlier key; minting one in ${secretKeyStorePath}."
|
||||
fi
|
||||
if ! printf '%s' "$candidate" | bao kv put -cas=0 ${lib.escapeShellArg secretKeyStorePath} value=- >/dev/null 2>"$err"; then
|
||||
echo "could not create ${secretKeyStorePath}; reading it back in case another run created it." >&2
|
||||
cat "$err" >&2
|
||||
fi
|
||||
if ! key="$(bao kv get -field=value ${lib.escapeShellArg secretKeyStorePath} 2>"$err")"; then
|
||||
cat "$err" >&2
|
||||
use_earlier_key "swarm-bao did not return ${secretKeyStorePath}"
|
||||
fi
|
||||
fi
|
||||
if [ -z "$key" ]; then
|
||||
echo "swarm-bao returned an empty ${secretKeyStorePath}; leaving the file as it is." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
land "$key"
|
||||
'';
|
||||
};
|
||||
|
||||
# swarm-controller reads Grafana's alerts with a token of a Viewer service
|
||||
# account, created here through Grafana's API as the built-in admin and
|
||||
# handed over through the store.
|
||||
|
|
@ -1253,55 +1376,6 @@ in
|
|||
};
|
||||
systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames storeFiles;
|
||||
|
||||
# Grafana's `secret_key` has **no default in nixpkgs** and an
|
||||
# assertion refuses the build without one — which is how the first
|
||||
# version of this module broke a deploy. It signs the datasource
|
||||
# secrets in Grafana's own database.
|
||||
#
|
||||
# Generated in-container on first boot, like authelia's own keys and
|
||||
# for the same reason: nothing outside this container ever reads it,
|
||||
# which makes in-container generation right rather than merely
|
||||
# easier. (The OIDC client secret is delivered host-side precisely
|
||||
# because it has a second reader.)
|
||||
#
|
||||
# ⚠️ Generated ONCE and kept — the `-s` guard is load-bearing.
|
||||
# Rotating this key does not re-encrypt what it already encrypted, so
|
||||
# a fresh key on every boot would leave Grafana unable to decrypt its
|
||||
# own stored datasource secrets. Under /var/lib, never /run.
|
||||
systemd.services.swarm-grafana-secret-key = {
|
||||
description = "generate Grafana's datasource encryption key on first boot";
|
||||
wantedBy = [ "grafana.service" ];
|
||||
before = [ "grafana.service" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
# Same User/Group/StateDirectory as grafana itself, so systemd
|
||||
# creates the directory owned by the account that has to read
|
||||
# the file — no chown, no mode juggling.
|
||||
User = "grafana";
|
||||
Group = "grafana";
|
||||
StateDirectory = "grafana-secret";
|
||||
StateDirectoryMode = "0700";
|
||||
SyslogIdentifier = "swarm-grafana-secret-key";
|
||||
};
|
||||
path = [
|
||||
pkgs.openssl
|
||||
pkgs.coreutils
|
||||
];
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
${atomicWriteSecret}
|
||||
|
||||
key=${lib.escapeShellArg secretKeyPath}
|
||||
if [ ! -s "$key" ]; then
|
||||
value=$(openssl rand -hex 32)
|
||||
atomic_write_secret 0600 "" "$key" "$value"
|
||||
echo "generated Grafana's datasource encryption key"
|
||||
fi
|
||||
'';
|
||||
};
|
||||
|
||||
# Upstream applies `security.admin_password` only when it creates the
|
||||
# database, so a database created before that setting keeps the
|
||||
# password it was created with. This resets it from the delivered
|
||||
|
|
@ -1418,7 +1492,8 @@ in
|
|||
|
||||
# ⚠️ `$__file{}`, and required rather than optional: nixpkgs
|
||||
# dropped the default and asserts on null, so without this the
|
||||
# whole host fails to build. The unit below generates it.
|
||||
# whole host fails to build. Written by `swarm-bao-grafana-secret-key`
|
||||
# on the host.
|
||||
security.secret_key = "$__file{${secretKeyPath}}";
|
||||
|
||||
# The password a newly created database gets; the `preStart`
|
||||
|
|
|
|||
Loading…
Reference in a new issue