read grafana's datasource key from the store into /run, seeding an empty store from the old file
This commit is contained in:
atlas
parent
fcc9c8d0ba
commit
204abb1d0e
6 changed files with 236 additions and 69 deletions
|
|
@ -93,6 +93,10 @@ in
|
|||
# what is minted, seeded from a leftover file or left alone when the store
|
||||
# cannot answer.
|
||||
script-test-oidc-client-secrets = import ./script-tests/oidc-client-secrets.nix { inherit pkgs; };
|
||||
# Runs ./host-modules/lib/grafana-secret-key.nix's decision against a stub
|
||||
# `bao`: a store that holds a key wins, an empty one takes the seed, and an
|
||||
# unreachable one mints nothing.
|
||||
script-test-grafana-secret-key = import ./script-tests/grafana-secret-key.nix { inherit pkgs; };
|
||||
module-eval-nats-tls = import ./module-eval/nats-tls.nix {
|
||||
inherit pkgs self nixosSystem;
|
||||
inherit (pkgs) lib;
|
||||
|
|
|
|||
58
nix/host-modules/lib/grafana-secret-key.nix
Normal file
58
nix/host-modules/lib/grafana-secret-key.nix
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
# The store's value for Grafana's datasource-encryption key, seeded once from
|
||||
# the file an earlier release generated in the container.
|
||||
#
|
||||
# Grafana cannot read datasource secrets encrypted with another key, so a store
|
||||
# with no value takes the file's, and a new key is minted only when there is no
|
||||
# file either. The store is written with `-cas=0`, so a value it holds is never
|
||||
# replaced and a lost race reads the winner's back. The file stays, unused.
|
||||
#
|
||||
# `grafana_secret_key` sets `GRAFANA_KEY` and returns 0, or returns 1 when the
|
||||
# store does not log in, answer or take a value; nothing is minted then.
|
||||
#
|
||||
# Pure function — NOT a NixOS module. Call it from a unit's `script`, with
|
||||
# `bao` and `coreutils` on its `path` and the `BAO_*` client environment set:
|
||||
#
|
||||
# script = ''
|
||||
# ${import ./lib/grafana-secret-key.nix { }}
|
||||
# grafana_secret_key secret/swarm/services/foo/secret-key /path/to/seed openssl rand -hex 32 || exit 1
|
||||
# '';
|
||||
{ }:
|
||||
''
|
||||
# <store leaf> <seed file> <mint command…>
|
||||
grafana_secret_key() {
|
||||
local leaf="$1" seed="$2" candidate err
|
||||
shift 2
|
||||
GRAFANA_KEY=""
|
||||
err="$(mktemp)"
|
||||
|
||||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||||
cat "$err" >&2
|
||||
rm -f "$err"
|
||||
echo "could not log in to the swarm secret store." >&2
|
||||
return 1
|
||||
fi
|
||||
export BAO_TOKEN
|
||||
|
||||
if ! GRAFANA_KEY="$(bao kv get -field=value "$leaf" 2>"$err")" || [ -z "$GRAFANA_KEY" ]; then
|
||||
if [ -s "$seed" ]; then
|
||||
candidate="$(tr -d '\n' < "$seed")"
|
||||
echo "storing the key at $seed in $leaf."
|
||||
else
|
||||
candidate="$("$@")"
|
||||
echo "no earlier key; minting one in $leaf."
|
||||
fi
|
||||
if ! printf '%s' "$candidate" | bao kv put -cas=0 "$leaf" value=- >/dev/null 2>"$err"; then
|
||||
cat "$err" >&2
|
||||
echo "could not create $leaf; reading it back in case another run created it." >&2
|
||||
fi
|
||||
if ! GRAFANA_KEY="$(bao kv get -field=value "$leaf" 2>"$err")" || [ -z "$GRAFANA_KEY" ]; then
|
||||
cat "$err" >&2
|
||||
rm -f "$err"
|
||||
echo "the store did not return $leaf." >&2
|
||||
GRAFANA_KEY=""
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
rm -f "$err"
|
||||
}
|
||||
''
|
||||
|
|
@ -369,8 +369,8 @@ let
|
|||
# Grafana unable to read its stored datasource secrets.
|
||||
secretKeyStorePath = "secret/swarm/services/${cfg.oidc.clientId}/secret-key";
|
||||
secretKeyPath = "${secretDir}/secret-key";
|
||||
# The earlier file as the host sees it. Never removed: it is the seed, and
|
||||
# the key Grafana runs on while the store's policy does not grant the leaf.
|
||||
# The earlier file as the host sees it: the seed for an empty store. It stays
|
||||
# on disk, unused once the store holds the key.
|
||||
legacySecretKeyPath = "/var/lib/nixos-containers/${cfg.machine}/var/lib/grafana-secret/secret_key";
|
||||
|
||||
# The built-in `admin` account's password. Grafana's API takes basic auth
|
||||
|
|
@ -411,6 +411,7 @@ let
|
|||
|
||||
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
|
||||
refreshConsumer = import ./lib/refresh-consumer.nix { };
|
||||
grafanaSecretKey = import ./lib/grafana-secret-key.nix { };
|
||||
|
||||
# First statement of each delivery script, ahead of the store login: the
|
||||
# directory is the container's bind source, so it has to exist even when the
|
||||
|
|
@ -1035,12 +1036,11 @@ in
|
|||
# first, and a new key only when there is no earlier file either. Same
|
||||
# shape and grant as the admin password above.
|
||||
#
|
||||
# The key never changes. An earlier release generated it in the container
|
||||
# on first boot and kept it, so a host that has run Grafana stores that
|
||||
# same key. While the store's policy does not grant the leaf yet (the
|
||||
# store's host is deployed after this one) or the store is unreachable,
|
||||
# the earlier file is what Grafana runs on, and the unit succeeds; a host
|
||||
# without one has no key to keep, so there the unit fails and retries.
|
||||
# The key never changes. An earlier release generated it in the container on
|
||||
# first boot, so a host that has run Grafana stores that same key
|
||||
# (`grafana_secret_key`, ./lib/grafana-secret-key.nix). The earlier file is
|
||||
# only a seed: with the store down or not granting the leaf the unit fails
|
||||
# and retries, and Grafana does not start without the key in /run.
|
||||
systemd.services.swarm-bao-grafana-secret-key = {
|
||||
description = "create Grafana's datasource encryption key in the swarm secret store if absent and fetch it";
|
||||
# Ordered after the store where it runs here:
|
||||
|
|
@ -1079,70 +1079,17 @@ in
|
|||
|
||||
${ensureSecretDir}
|
||||
|
||||
err="$(mktemp)"
|
||||
trap 'rm -f "$err"' EXIT
|
||||
${grafanaSecretKey}
|
||||
|
||||
# The earlier file's key, or nothing.
|
||||
earlier_key() {
|
||||
if [ -s ${lib.escapeShellArg legacySecretKeyPath} ]; then
|
||||
tr -d '\n' < ${lib.escapeShellArg legacySecretKeyPath}
|
||||
fi
|
||||
}
|
||||
grafana_secret_key ${lib.escapeShellArg secretKeyStorePath} ${lib.escapeShellArg legacySecretKeyPath} openssl rand -hex 32 || exit 1
|
||||
|
||||
# Owner and mode as for the OIDC secret above, for its reasons.
|
||||
# `$__file{}` is read as Grafana starts, so a changed file needs a
|
||||
# restart.
|
||||
land() {
|
||||
if secret_differs ${lib.escapeShellArg secretKeyPath} "$1"; then
|
||||
atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg secretKeyPath} "$1"
|
||||
fi
|
||||
refresh_consumer ${lib.escapeShellArg cfg.machine} grafana.service ${lib.escapeShellArg secretKeyPath}
|
||||
}
|
||||
|
||||
# The store cannot answer: run on the earlier file's key, or fail.
|
||||
use_earlier_key() {
|
||||
local key
|
||||
key="$(earlier_key)"
|
||||
if [ -z "$key" ]; then
|
||||
echo "$1, and there is no earlier key at ${legacySecretKeyPath}; Grafana does not start without one." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "$1; running Grafana on the key at ${legacySecretKeyPath}." >&2
|
||||
land "$key"
|
||||
exit 0
|
||||
}
|
||||
|
||||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||||
cat "$err" >&2
|
||||
use_earlier_key "could not log in to swarm-bao with Grafana's certificate"
|
||||
if secret_differs ${lib.escapeShellArg secretKeyPath} "$GRAFANA_KEY"; then
|
||||
atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg secretKeyPath} "$GRAFANA_KEY"
|
||||
fi
|
||||
export BAO_TOKEN
|
||||
|
||||
# Read, create on a miss, read again: the admin password's sequence
|
||||
# above, for its reasons, with the earlier key as the first candidate.
|
||||
if ! key="$(bao kv get -field=value ${lib.escapeShellArg secretKeyStorePath} 2>"$err")"; then
|
||||
candidate="$(earlier_key)"
|
||||
if [ -n "$candidate" ]; then
|
||||
echo "storing the key at ${legacySecretKeyPath} in ${secretKeyStorePath}."
|
||||
else
|
||||
candidate="$(openssl rand -hex 32)"
|
||||
echo "no earlier key; minting one in ${secretKeyStorePath}."
|
||||
fi
|
||||
if ! printf '%s' "$candidate" | bao kv put -cas=0 ${lib.escapeShellArg secretKeyStorePath} value=- >/dev/null 2>"$err"; then
|
||||
echo "could not create ${secretKeyStorePath}; reading it back in case another run created it." >&2
|
||||
cat "$err" >&2
|
||||
fi
|
||||
if ! key="$(bao kv get -field=value ${lib.escapeShellArg secretKeyStorePath} 2>"$err")"; then
|
||||
cat "$err" >&2
|
||||
use_earlier_key "swarm-bao did not return ${secretKeyStorePath}"
|
||||
fi
|
||||
fi
|
||||
if [ -z "$key" ]; then
|
||||
echo "swarm-bao returned an empty ${secretKeyStorePath}; leaving the file as it is." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
land "$key"
|
||||
refresh_consumer ${lib.escapeShellArg cfg.machine} grafana.service ${lib.escapeShellArg secretKeyPath}
|
||||
'';
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -268,8 +268,9 @@ let
|
|||
guest = m.containers.swarm-grafana.config;
|
||||
earlier = "/var/lib/nixos-containers/swarm-grafana/var/lib/grafana-secret/secret_key";
|
||||
in
|
||||
lib.hasInfix "bao kv put -cas=0 secret/swarm/services/swarm-grafana/secret-key value=-" unit.script
|
||||
&& lib.hasInfix "tr -d '\\n' < ${earlier}" unit.script
|
||||
lib.hasInfix "bao kv put -cas=0 \"\$leaf\" value=-" unit.script
|
||||
&& lib.hasInfix "grafana_secret_key secret/swarm/services/swarm-grafana/secret-key ${earlier} openssl rand -hex 32" unit.script
|
||||
&& lib.length (lib.splitString earlier unit.script) == 2
|
||||
&& lib.hasInfix "atomic_write_secret 0400 196:0 /run/swarm-grafana-secrets/secret-key" unit.script
|
||||
&& lib.elem "container@swarm-grafana.service" unit.before
|
||||
&& unit.environment.BAO_CLIENT_CERT == "/etc/pki/bao-grafana-oidc.pem"
|
||||
|
|
|
|||
68
nix/script-tests/grafana-secret-key.nix
Normal file
68
nix/script-tests/grafana-secret-key.nix
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
# `checks.script-test-grafana-secret-key` — runs
|
||||
# ../host-modules/lib/grafana-secret-key.nix's `grafana_secret_key` against a
|
||||
# stub `bao` that keeps its leaves in files. The cases are in
|
||||
# ./grafana-secret-key.sh.
|
||||
#
|
||||
# What runs is the rendered helper, sourced as the unit's script includes it.
|
||||
{
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
# `login` answers from `FAKE_BAO_LOGIN_RC`. `kv get` and `kv put -cas=0` act
|
||||
# on a file per leaf under `FAKE_BAO_KV`, and answer as bao does: a missing
|
||||
# leaf is "No value found at <path>" on stderr, status 2. `FAKE_BAO_DENY_READ`
|
||||
# and `FAKE_BAO_DENY_WRITE` answer 403 as a policy that grants neither does.
|
||||
# `FAKE_BAO_RACE` has another run create the leaf with that value between
|
||||
# this one's read and its write. Every call is logged without the value
|
||||
# `put` reads from stdin.
|
||||
fakeBao = pkgs.writeShellScriptBin "bao" ''
|
||||
echo "$*" >> "$FAKE_BAO_LOG"
|
||||
case "$1 $2" in
|
||||
"login -method=cert")
|
||||
[ "$FAKE_BAO_LOGIN_RC" = 0 ] || { echo "fake bao: certificate refused" >&2; exit 2; }
|
||||
printf 's.fake'
|
||||
;;
|
||||
"kv get")
|
||||
[ "''${BAO_TOKEN-}" = s.fake ] || { echo "fake bao: kv without the login's token" >&2; exit 97; }
|
||||
[ "$3" = -field=value ] || { echo "fake bao: get without -field=value" >&2; exit 95; }
|
||||
leaf="$FAKE_BAO_KV/$(printf '%s' "$4" | tr / _)"
|
||||
[ -z "''${FAKE_BAO_DENY_READ-}" ] || { echo "fake bao: permission denied" >&2; exit 2; }
|
||||
if [ -n "''${FAKE_BAO_RACE-}" ] && [ ! -e "$FAKE_BAO_KV/.raced" ]; then
|
||||
touch "$FAKE_BAO_KV/.raced"
|
||||
printf '%s' "$FAKE_BAO_RACE" > "$leaf"
|
||||
echo "No value found at secret/data/x" >&2
|
||||
exit 2
|
||||
fi
|
||||
[ -e "$leaf" ] || { echo "No value found at secret/data/x" >&2; exit 2; }
|
||||
cat "$leaf"
|
||||
echo
|
||||
;;
|
||||
"kv put")
|
||||
[ "''${BAO_TOKEN-}" = s.fake ] || { echo "fake bao: kv without the login's token" >&2; exit 97; }
|
||||
[ "$3" = -cas=0 ] || { echo "fake bao: put without -cas=0" >&2; exit 96; }
|
||||
[ "$5" = value=- ] || { echo "fake bao: put with the value as an argument" >&2; exit 94; }
|
||||
leaf="$FAKE_BAO_KV/$(printf '%s' "$4" | tr / _)"
|
||||
[ -z "''${FAKE_BAO_DENY_WRITE-}" ] || { echo "fake bao: permission denied" >&2; exit 2; }
|
||||
[ ! -e "$leaf" ] || { echo "fake bao: check-and-set parameter did not match" >&2; exit 2; }
|
||||
cat > "$leaf"
|
||||
;;
|
||||
*)
|
||||
echo "fake bao: unexpected call: $*" >&2
|
||||
exit 98
|
||||
;;
|
||||
esac
|
||||
'';
|
||||
in
|
||||
pkgs.runCommand "hyperhive-script-test-grafana-secret-key"
|
||||
{
|
||||
HELPER = pkgs.writeText "grafana-secret-key.sh" (
|
||||
import ../host-modules/lib/grafana-secret-key.nix { }
|
||||
);
|
||||
FAKE_BAO_BIN = "${fakeBao}/bin";
|
||||
nativeBuildInputs = [ pkgs.coreutils ];
|
||||
}
|
||||
''
|
||||
${pkgs.bash}/bin/bash ${./grafana-secret-key.sh}
|
||||
touch "$out"
|
||||
''
|
||||
89
nix/script-tests/grafana-secret-key.sh
Normal file
89
nix/script-tests/grafana-secret-key.sh
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
# Cases for ./grafana-secret-key.nix. Each case sets up the store's leaf, the
|
||||
# seed file and the stub's behaviour, runs `grafana_secret_key` the way the
|
||||
# unit's script does, and compares its status, the key it hands over and what
|
||||
# the store holds afterwards.
|
||||
set -uo pipefail
|
||||
|
||||
failures=0
|
||||
leaf=secret/swarm/services/t/secret-key
|
||||
|
||||
# setup <description>
|
||||
setup() {
|
||||
case=$1
|
||||
dir=$(mktemp -d)
|
||||
mkdir "$dir/kv"
|
||||
export FAKE_BAO_KV=$dir/kv FAKE_BAO_LOG=$dir/bao.log FAKE_BAO_LOGIN_RC=0
|
||||
unset FAKE_BAO_DENY_READ FAKE_BAO_DENY_WRITE FAKE_BAO_RACE
|
||||
: >"$FAKE_BAO_LOG"
|
||||
seed=$dir/secret_key
|
||||
stored=$FAKE_BAO_KV/$(printf '%s' "$leaf" | tr / _)
|
||||
}
|
||||
|
||||
mint() {
|
||||
printf 'minted-key\n'
|
||||
}
|
||||
|
||||
# expect <status> <key> <what the store holds, "" for nothing>
|
||||
expect() {
|
||||
(
|
||||
PATH="$FAKE_BAO_BIN:$PATH"
|
||||
unset BAO_TOKEN
|
||||
# shellcheck source=/dev/null
|
||||
. "$HELPER"
|
||||
rc=0
|
||||
grafana_secret_key "$leaf" "$seed" mint >/dev/null 2>"$dir/err" || rc=$?
|
||||
printf '%s' "$rc" >"$dir/rc"
|
||||
printf '%s' "$GRAFANA_KEY" >"$dir/key"
|
||||
)
|
||||
held=$(cat "$stored" 2>/dev/null)
|
||||
if [ "$(cat "$dir/rc")" != "$1" ]; then fail "exited $(cat "$dir/rc"), wanted $1: $(cat "$dir/err")"; fi
|
||||
if [ "$(cat "$dir/key")" != "$2" ]; then fail "handed over '$(cat "$dir/key")', wanted '$2'"; fi
|
||||
if [ "$held" != "$3" ]; then fail "the store holds '$held', wanted '$3'"; fi
|
||||
}
|
||||
|
||||
fail() {
|
||||
echo "FAILED: $case: $*" >&2
|
||||
failures=$((failures + 1))
|
||||
}
|
||||
|
||||
setup "an empty store takes the seed's key"
|
||||
printf 'seed-key\n' >"$seed"
|
||||
expect 0 seed-key seed-key
|
||||
grep -q '^kv put -cas=0 ' "$FAKE_BAO_LOG" || fail "no create-only write"
|
||||
[ -e "$seed" ] || fail "the seed file was removed"
|
||||
|
||||
setup "a store that holds a key wins over the seed, and is not written"
|
||||
printf 'seed-key\n' >"$seed"
|
||||
printf 'store-key' >"$stored"
|
||||
expect 0 store-key store-key
|
||||
! grep -q '^kv put' "$FAKE_BAO_LOG" || fail "wrote to the store"
|
||||
|
||||
setup "with no seed and an empty store, a key is minted and stored"
|
||||
expect 0 minted-key minted-key
|
||||
|
||||
setup "a store that refuses the login mints nothing, with or without a seed"
|
||||
export FAKE_BAO_LOGIN_RC=1
|
||||
expect 1 "" ""
|
||||
printf 'seed-key\n' >"$seed"
|
||||
expect 1 "" ""
|
||||
|
||||
setup "a store that refuses the read stores and hands over nothing"
|
||||
printf 'seed-key\n' >"$seed"
|
||||
export FAKE_BAO_DENY_READ=1
|
||||
expect 1 "" seed-key
|
||||
|
||||
setup "a store that refuses the write hands over nothing"
|
||||
export FAKE_BAO_DENY_WRITE=1
|
||||
expect 1 "" ""
|
||||
|
||||
setup "a store that holds an empty value hands over nothing"
|
||||
: >"$stored"
|
||||
expect 1 "" ""
|
||||
|
||||
setup "a lost race takes the winner's key"
|
||||
printf 'seed-key\n' >"$seed"
|
||||
export FAKE_BAO_RACE=other-key
|
||||
expect 0 other-key other-key
|
||||
|
||||
echo "$failures failures"
|
||||
[ "$failures" = 0 ]
|
||||
Loading…
Reference in a new issue