fetch the secrets unit's store leaves concurrently and log how long each phase takes
This commit is contained in:
atlas
parent
23c2527199
commit
197caa1e52
7 changed files with 268 additions and 9 deletions
|
|
@ -27,7 +27,7 @@
|
|||
client_secret_read() {
|
||||
CLIENT_SECRET_VALUE=""
|
||||
[ "$STORE_UP" = 1 ] || return 2
|
||||
if CLIENT_SECRET_VALUE="$(store_bao kv get -field=value "$1" 2>"$STORE_ERR")" \
|
||||
if CLIENT_SECRET_VALUE="$(store_read "$1")" \
|
||||
&& [ -n "$CLIENT_SECRET_VALUE" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -8,6 +8,11 @@
|
|||
# race reads the winner's back. A read of the leaf that fails for any reason
|
||||
# but "No value found" prints the store's error and neither mints nor writes.
|
||||
#
|
||||
# `store_prefetch` reads many leaves concurrently, and the first `store_read` of
|
||||
# a leaf returns that answer, status and error included, instead of asking
|
||||
# again, so what a caller decides from a read is the same. Later reads of the
|
||||
# leaf, such as one that checks a write, ask the store.
|
||||
#
|
||||
# Pure function — NOT a NixOS module. Call it from a module's `let`:
|
||||
#
|
||||
# storeSeededSecret = import ./lib/store-seeded-secret.nix { };
|
||||
|
|
@ -31,7 +36,8 @@
|
|||
store_init() {
|
||||
STORE_ERR="$(mktemp)"
|
||||
STORE_UP=0
|
||||
trap 'rm -f "$STORE_ERR"' EXIT
|
||||
STORE_PREFETCH="$(mktemp -d)"
|
||||
trap 'rm -rf "$STORE_ERR" "$STORE_PREFETCH"' EXIT
|
||||
}
|
||||
|
||||
# Sets STORE_UP. A store that cannot be logged in to leaves every secret to
|
||||
|
|
@ -48,6 +54,41 @@
|
|||
fi
|
||||
}
|
||||
|
||||
# <leaf>… Starts the first read of each leaf now, at most STORE_PREFETCH_JOBS
|
||||
# at a time, and returns when all have ended. Each read is bounded by
|
||||
# `store_bao`, and its answer is kept for `store_read`. Does nothing without
|
||||
# a login.
|
||||
store_prefetch() {
|
||||
local leaf key
|
||||
[ "$STORE_UP" = 1 ] || return 0
|
||||
for leaf in "$@"; do
|
||||
while [ "$(jobs -rp | wc -l)" -ge "''${STORE_PREFETCH_JOBS:-8}" ]; do
|
||||
wait -n || true
|
||||
done
|
||||
key="$STORE_PREFETCH/$(printf '%s' "$leaf" | tr / _)"
|
||||
(
|
||||
rc=0
|
||||
store_bao kv get -field=value "$leaf" >"$key.out" 2>"$key.err" || rc=$?
|
||||
echo "$rc" >"$key.rc"
|
||||
) &
|
||||
done
|
||||
wait
|
||||
}
|
||||
|
||||
# <leaf>. `store_bao kv get -field=value <leaf>`, with the error in
|
||||
# $STORE_ERR: the answer `store_prefetch` kept, once, then the store's.
|
||||
store_read() {
|
||||
local key="$STORE_PREFETCH/$(printf '%s' "$1" | tr / _)" rc
|
||||
if [ -e "$key.rc" ]; then
|
||||
rc="$(< "$key.rc")"
|
||||
cat "$key.out"
|
||||
cat "$key.err" >"$STORE_ERR"
|
||||
rm -f "$key.out" "$key.err" "$key.rc"
|
||||
return "$rc"
|
||||
fi
|
||||
store_bao kv get -field=value "$1" 2>"$STORE_ERR"
|
||||
}
|
||||
|
||||
# <leaf> <earlier file> <mint command…>. The value on stdout; status 1 when
|
||||
# neither the store nor the file can give one. `kv get` exits 2 both for a
|
||||
# leaf that holds nothing ("No value found at <path>") and for a refused
|
||||
|
|
@ -58,7 +99,7 @@
|
|||
local leaf="$1" earlier="$2" value candidate rc=0
|
||||
shift 2
|
||||
if [ "$STORE_UP" = 1 ]; then
|
||||
value="$(store_bao kv get -field=value "$leaf" 2>"$STORE_ERR")" || rc=$?
|
||||
value="$(store_read "$leaf")" || rc=$?
|
||||
if [ "$rc" = 0 ] && [ -n "$value" ]; then
|
||||
printf '%s' "$value"
|
||||
return 0
|
||||
|
|
|
|||
|
|
@ -526,12 +526,14 @@ let
|
|||
# cannot be read fails the run too, after the files are rendered from the
|
||||
# digests (`STORE_UNREADABLE`, checked at the end of the unit's script), so
|
||||
# the unit retries until the store answers and authelia has started from the
|
||||
# digests meanwhile.
|
||||
# digests meanwhile. Runs after `storeKeysScript`, which defines
|
||||
# `phase_done` and has already fetched these leaves.
|
||||
storeMintScript = ''
|
||||
${oidcClientSecrets}
|
||||
${lib.concatMapStrings (c: ''
|
||||
mint_client_secret ${lib.escapeShellArg c.id} ${lib.escapeShellArg (clientLeaf c.id)} || { rc=$?; [ "$rc" = 2 ] || exit "$rc"; STORE_UNREADABLE=1; }
|
||||
'') secretClients}
|
||||
phase_done clients
|
||||
'';
|
||||
|
||||
# The OIDC half of the first-boot generator, kept out of the script
|
||||
|
|
@ -624,6 +626,12 @@ let
|
|||
done
|
||||
'';
|
||||
|
||||
# Every leaf the unit reads, for `store_prefetch`: the keys and the client
|
||||
# secrets. The reads that follow are answered from what it fetched.
|
||||
storeLeaves =
|
||||
map keyLeaf (randomKeys ++ lib.optional oidcEnabled "oidc-issuer")
|
||||
++ lib.optionals oidcEnabled (map (c: clientLeaf c.id) secretClients);
|
||||
|
||||
# The same keys from the store, at the same paths. The store's value wins,
|
||||
# then the file's, which seeds the store when it holds none, so a key never
|
||||
# changes value by moving; a new one is minted only where neither exists.
|
||||
|
|
@ -632,8 +640,21 @@ let
|
|||
storeKeysScript = ''
|
||||
${storeSeededSecret}
|
||||
${refreshConsumer}
|
||||
|
||||
# <name>. One line: how long the phase that ends here took, so a slow store
|
||||
# shows where the time went. Not `SECONDS`, which counts whole seconds.
|
||||
PHASE_AT="$EPOCHREALTIME"
|
||||
phase_done() {
|
||||
local now="$EPOCHREALTIME"
|
||||
echo "phase $1: $(( (''${now//[.,]/} - ''${PHASE_AT//[.,]/}) / 1000 )) ms"
|
||||
PHASE_AT="$now"
|
||||
}
|
||||
|
||||
store_init
|
||||
store_login
|
||||
phase_done login
|
||||
store_prefetch ${lib.concatMapStringsSep " " lib.escapeShellArg storeLeaves}
|
||||
phase_done reads
|
||||
|
||||
mint_hex() {
|
||||
head -c 64 /dev/urandom | od -An -tx1 | tr -d ' \n'
|
||||
|
|
@ -661,6 +682,7 @@ let
|
|||
keep_key "$f" mint_hex
|
||||
done
|
||||
${lib.optionalString oidcEnabled "keep_key oidc-issuer mint_issuer"}
|
||||
phase_done keys
|
||||
'';
|
||||
|
||||
# A netns of its own on the hive bridge: the container dials the store, the
|
||||
|
|
@ -1197,6 +1219,10 @@ in
|
|||
# accepts a connection and never answers: this unit is in the
|
||||
# boot wait of a container with a one-minute start limit.
|
||||
# What a killed run left is picked up by the next one.
|
||||
#
|
||||
# The run's store reads are concurrent (`store_prefetch`): its
|
||||
# length is the login, a few rounds of reads and the writes,
|
||||
# not one round trip per leaf. Raising this raises that wait.
|
||||
TimeoutStartSec = 30;
|
||||
RestartSec = 15;
|
||||
RestartSteps = 3;
|
||||
|
|
|
|||
|
|
@ -529,6 +529,56 @@ let
|
|||
&& !(onDisk.environment ? BAO_ADDR)
|
||||
&& !(onDisk.serviceConfig ? LoadCredential);
|
||||
}
|
||||
{
|
||||
# The unit runs inside the container's boot wait, so its start budget is
|
||||
# the ceiling of that wait and stays at 30 s; what fits the budget is that
|
||||
# every leaf the run reads is fetched together right after the login, then
|
||||
# the sequential reads are answered from that. The leaves are those of
|
||||
# the keys and of every `mint_client_secret` call, found by the same
|
||||
# lookup, so a leaf added to either and left out of the prefetch fails
|
||||
# here. The controls: the no-identity unit has no prefetch, and a call
|
||||
# with its leaf left out would not be found by the lookup.
|
||||
name = "the secrets unit fetches every leaf it reads in one concurrent pass after the login, inside its 30 s budget";
|
||||
ok =
|
||||
let
|
||||
unit = m: (inAuthelia m).systemd.services.authelia-swarm-secrets;
|
||||
s = (unit secretPublisherHere).script;
|
||||
lines = lib.splitString "\n" s;
|
||||
calls = lib.filter (l: lib.hasPrefix "store_prefetch secret/" l) lines;
|
||||
prefetch = lib.head calls;
|
||||
leafOf = l: builtins.match "mint_client_secret [^ ]+ ([^ ]+) .*" l;
|
||||
clientLeaves = lib.concatMap (
|
||||
l:
|
||||
let
|
||||
m = leafOf l;
|
||||
in
|
||||
lib.optional (m != null) (lib.head m)
|
||||
) lines;
|
||||
hasLeaf = leaf: lib.elem leaf (lib.splitString " " prefetch);
|
||||
phases = lib.filter (lib.hasPrefix "phase_done ") lines;
|
||||
in
|
||||
(unit secretPublisherHere).serviceConfig.TimeoutStartSec == 30
|
||||
&& lib.length calls == 1
|
||||
&& lib.length clientLeaves >= 4
|
||||
&& lib.all hasLeaf clientLeaves
|
||||
&& lib.all (k: hasLeaf "secret/swarm/services/swarm-authelia/keys/${k}") [
|
||||
"jwt"
|
||||
"session"
|
||||
"storage-encryption"
|
||||
"oidc-hmac"
|
||||
"oidc-issuer"
|
||||
]
|
||||
&& lib.length (lib.splitString " " prefetch) == lib.length clientLeaves + 6
|
||||
&&
|
||||
phases == [
|
||||
"phase_done login"
|
||||
"phase_done reads"
|
||||
"phase_done keys"
|
||||
"phase_done clients"
|
||||
]
|
||||
&& !(hasLeaf "secret/swarm/services/not-a-leaf")
|
||||
&& !(lib.hasInfix "\nstore_prefetch secret/" (unit secretPublisherNoIdentity).script);
|
||||
}
|
||||
{
|
||||
# The leaf crosses into the container as nspawn system credentials, and
|
||||
# the unit loads them and names them in its environment. On the store's
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ setup() {
|
|||
mkdir "$dir/kv" "$dir/clients"
|
||||
export FAKE_BAO_KV=$dir/kv FAKE_BAO_LOG=$dir/bao.log FAKE_BAO_LOGIN_RC=0
|
||||
export FAKE_AUTHELIA_LOG=$dir/authelia.log
|
||||
unset FAKE_BAO_DENY_READ FAKE_BAO_DENY_WRITE FAKE_BAO_RACE FAKE_AUTHELIA_EMPTY FAKE_AUTHELIA_NODIGEST
|
||||
unset FAKE_BAO_DENY_READ FAKE_BAO_DENY_WRITE FAKE_BAO_RACE FAKE_AUTHELIA_EMPTY FAKE_AUTHELIA_NODIGEST PREFETCH
|
||||
: >"$FAKE_BAO_LOG"
|
||||
: >"$FAKE_AUTHELIA_LOG"
|
||||
clients=$dir/clients
|
||||
|
|
@ -34,7 +34,8 @@ setup() {
|
|||
}
|
||||
|
||||
# run <function> — the function for $id and $leaf, with the stubs first on
|
||||
# PATH, after the store login the units make first. Sets $rc and $minted.
|
||||
# PATH, after the store login the units make first and, when $PREFETCH is set,
|
||||
# `store_prefetch` of the leaf. Sets $rc and $minted.
|
||||
run() {
|
||||
minted=$(
|
||||
set -e
|
||||
|
|
@ -45,6 +46,7 @@ run() {
|
|||
clients=$dir/clients
|
||||
store_init
|
||||
store_login 2>>"$dir/err"
|
||||
[ -z "${PREFETCH-}" ] || store_prefetch "$leaf" 2>>"$dir/err"
|
||||
rc=0
|
||||
"$1" "$id" "$leaf" 2>>"$dir/err" >>"$dir/out" || rc=$?
|
||||
echo "${MINTED-}"
|
||||
|
|
@ -262,6 +264,52 @@ if [ "$(id -u)" != 0 ]; then
|
|||
digest_is ""
|
||||
fi
|
||||
|
||||
# kv_gets: how many `kv get` calls the stub logged.
|
||||
kv_gets() {
|
||||
grep -c '^kv get' "$FAKE_BAO_LOG" || true
|
||||
}
|
||||
|
||||
setup "prefetch: a client the store and the disk both hold is left alone after one read"
|
||||
PREFETCH=1
|
||||
printf 'old-secret' >"$stored"
|
||||
printf 'old-digest\n' >"$digest"
|
||||
run mint_client_secret
|
||||
expect 0
|
||||
held old-secret
|
||||
digest_is old-digest
|
||||
if ran_authelia; then fail "minted although the store held a value"; fi
|
||||
if [ "$(kv_gets)" != 1 ]; then fail "$(kv_gets) reads, wanted 1"; fi
|
||||
|
||||
setup "prefetch: a digest whose secret is gone from the store is minted again, and the read back asks the store"
|
||||
PREFETCH=1
|
||||
printf 'old-digest\n' >"$digest"
|
||||
run mint_client_secret
|
||||
expect 0
|
||||
held minted-secret
|
||||
digest_is '$pbkdf2-sha512$fake'
|
||||
if [ "$minted" != 1 ]; then fail "did not report the mint"; fi
|
||||
if [ "$(kv_gets)" != 2 ]; then fail "$(kv_gets) reads, wanted the prefetch and the read back"; fi
|
||||
|
||||
setup "prefetch: a store that refuses the read leaves the digest, mints nothing and asks to retry"
|
||||
PREFETCH=1
|
||||
printf 'old-digest\n' >"$digest"
|
||||
export FAKE_BAO_DENY_READ=1
|
||||
run mint_client_secret
|
||||
expect 2
|
||||
digest_is old-digest
|
||||
if ran_authelia; then fail "minted although the read was refused"; fi
|
||||
if grep -q '^kv put' "$FAKE_BAO_LOG"; then fail "wrote although the read was refused"; fi
|
||||
if ! logged "the store cannot be read; keeping its digest"; then fail "no keep line"; fi
|
||||
|
||||
setup "prefetch: no digest but a secret in the store gets the digest of that secret"
|
||||
PREFETCH=1
|
||||
printf 'old-secret' >"$stored"
|
||||
run mint_client_secret
|
||||
expect 0
|
||||
held old-secret
|
||||
digest_is digest-of-old-secret
|
||||
if ran_authelia; then fail "minted over the store's secret"; fi
|
||||
|
||||
if [ "$failures" != 0 ]; then
|
||||
echo "$failures of $count cases failed" >&2
|
||||
exit 1
|
||||
|
|
|
|||
|
|
@ -16,7 +16,9 @@ let
|
|||
# `FAKE_BAO_READ_5XX` is a server error on the read. A leaf nobody wrote is
|
||||
# "No value found at <path>", status 2, as `bao kv get` answers it.
|
||||
# `FAKE_BAO_RACE` has another run create the leaf between this one's read and
|
||||
# its write. Every call is logged without the value `put` reads from stdin.
|
||||
# its write. `FAKE_BAO_READ_DELAY` holds each read that long and records, in
|
||||
# `$FAKE_BAO_KV/.peak`, how many reads were in flight with it. Every call is
|
||||
# logged without the value `put` reads from stdin.
|
||||
fakeBao = pkgs.writeShellScriptBin "bao" ''
|
||||
echo "$*" >> "$FAKE_BAO_LOG"
|
||||
case "$1 $2" in
|
||||
|
|
@ -28,6 +30,12 @@ let
|
|||
[ "''${BAO_TOKEN-}" = s.fake ] || { echo "fake bao: kv without the login's token" >&2; exit 97; }
|
||||
leaf="$FAKE_BAO_KV/$(printf '%s' "$4" | tr / _)"
|
||||
[ -z "''${FAKE_BAO_READ_TIMEOUT-}" ] || exit 124
|
||||
if [ -n "''${FAKE_BAO_READ_DELAY-}" ]; then
|
||||
touch "$FAKE_BAO_KV/.inflight.$$"
|
||||
ls "$FAKE_BAO_KV"/.inflight.* | wc -l >> "$FAKE_BAO_KV/.peak"
|
||||
sleep "$FAKE_BAO_READ_DELAY"
|
||||
rm -f "$FAKE_BAO_KV/.inflight.$$"
|
||||
fi
|
||||
[ -z "''${FAKE_BAO_READ_5XX-}" ] || { echo "Error reading secret/data/x: Error making API request. Code: 503. Errors: * Vault is sealed" >&2; exit 2; }
|
||||
[ -z "''${FAKE_BAO_DENY_READ-}" ] || { echo "fake bao: permission denied" >&2; exit 2; }
|
||||
if [ -n "''${FAKE_BAO_RACE-}" ] && [ ! -e "$FAKE_BAO_KV/.raced" ]; then
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ setup() {
|
|||
dir=$(mktemp -d)
|
||||
mkdir "$dir/kv"
|
||||
export FAKE_BAO_KV=$dir/kv FAKE_BAO_LOG=$dir/bao.log FAKE_BAO_LOGIN_RC=0
|
||||
unset FAKE_BAO_DENY_READ FAKE_BAO_DENY_WRITE FAKE_BAO_RACE FAKE_BAO_READ_TIMEOUT FAKE_BAO_READ_5XX
|
||||
unset FAKE_BAO_DENY_READ FAKE_BAO_DENY_WRITE FAKE_BAO_RACE FAKE_BAO_READ_TIMEOUT FAKE_BAO_READ_5XX FAKE_BAO_READ_DELAY PREFETCH
|
||||
: >"$FAKE_BAO_LOG"
|
||||
earlier=$dir/earlier
|
||||
minted=$dir/minted
|
||||
|
|
@ -34,7 +34,8 @@ mint() {
|
|||
printf 'minted-value\n'
|
||||
}
|
||||
|
||||
# run — `store_seeded` for $leaf and $earlier with the stub first on PATH.
|
||||
# run — `store_seeded` for $leaf and $earlier with the stub first on PATH, after
|
||||
# `store_prefetch` of the leaf when $PREFETCH is set.
|
||||
# Sets $rc and $got.
|
||||
run() {
|
||||
got=$(
|
||||
|
|
@ -45,6 +46,7 @@ run() {
|
|||
. "$HELPER"
|
||||
store_init
|
||||
store_login 2>>"$dir/err"
|
||||
[ -z "${PREFETCH-}" ] || store_prefetch "$leaf" 2>>"$dir/err"
|
||||
store_seeded "$leaf" "$earlier" mint 2>"$dir/err"
|
||||
)
|
||||
rc=$?
|
||||
|
|
@ -212,6 +214,90 @@ run
|
|||
expect 1 ""
|
||||
if ran_mint; then fail "minted with no store to hold the value"; fi
|
||||
|
||||
# kv_gets: how many `kv get` calls the stub logged.
|
||||
kv_gets() {
|
||||
grep -c '^kv get' "$FAKE_BAO_LOG" || true
|
||||
}
|
||||
|
||||
setup "prefetch: a leaf the store holds is read once, and the result is the same"
|
||||
PREFETCH=1
|
||||
printf 'stored-value' >"$stored"
|
||||
printf 'old-value\n' >"$earlier"
|
||||
run
|
||||
expect 0 stored-value
|
||||
held stored-value
|
||||
no_write
|
||||
no_mint
|
||||
if [ "$(kv_gets)" != 1 ]; then fail "$(kv_gets) reads, wanted 1"; fi
|
||||
|
||||
setup "prefetch: a leaf the store lacks is seeded from the file as without it"
|
||||
PREFETCH=1
|
||||
printf 'old-value\n' >"$earlier"
|
||||
run
|
||||
expect 0 old-value
|
||||
held old-value
|
||||
if ran_mint; then fail "minted although the file held a value"; fi
|
||||
|
||||
setup "prefetch: a refused read uses the file's value, writes nothing and prints the refusal, as without it"
|
||||
PREFETCH=1
|
||||
printf 'old-value\n' >"$earlier"
|
||||
export FAKE_BAO_DENY_READ=1 FAKE_BAO_DENY_WRITE=1
|
||||
run
|
||||
expect 0 old-value
|
||||
held ""
|
||||
no_write
|
||||
no_mint
|
||||
err_has "could not read $leaf from the swarm secret store"
|
||||
err_has "permission denied"
|
||||
|
||||
setup "prefetch: a read that times out is not treated as an absent leaf"
|
||||
PREFETCH=1
|
||||
printf 'stored-value' >"$stored"
|
||||
export FAKE_BAO_READ_TIMEOUT=1
|
||||
run
|
||||
expect 1 ""
|
||||
held stored-value
|
||||
no_write
|
||||
no_mint
|
||||
err_has "(exit status 124)"
|
||||
|
||||
setup "prefetch: a store that refuses the login reads nothing"
|
||||
PREFETCH=1
|
||||
printf 'old-value\n' >"$earlier"
|
||||
export FAKE_BAO_LOGIN_RC=1
|
||||
run
|
||||
expect 0 old-value
|
||||
if grep -q '^kv' "$FAKE_BAO_LOG"; then fail "called kv after a refused login"; fi
|
||||
|
||||
setup "prefetch: many leaves are read together, at most 8 at once, each once, and a second read of one asks the store"
|
||||
export FAKE_BAO_READ_DELAY=0.3
|
||||
many=()
|
||||
for i in $(seq 1 20); do
|
||||
many+=("secret/swarm/many/$i")
|
||||
printf 'v%s' "$i" >"$FAKE_BAO_KV/$(printf '%s' "secret/swarm/many/$i" | tr / _)"
|
||||
done
|
||||
want=$(for i in $(seq 1 20); do printf 'v%s\n' "$i"; done)
|
||||
got=$(
|
||||
set -e
|
||||
PATH="$FAKE_BAO_BIN:$PATH"
|
||||
unset BAO_TOKEN
|
||||
# shellcheck source=/dev/null
|
||||
. "$HELPER"
|
||||
store_init
|
||||
store_login
|
||||
store_prefetch "${many[@]}"
|
||||
for l in "${many[@]}"; do
|
||||
store_read "$l"
|
||||
echo
|
||||
done
|
||||
store_read "${many[0]}"
|
||||
)
|
||||
if [ "$got" != "$want"$'\n'v1 ]; then fail "values differ: $got"; fi
|
||||
if [ "$(kv_gets)" != 21 ]; then fail "$(kv_gets) reads, wanted 20 and the second read of one"; fi
|
||||
peak=$(sort -n "$FAKE_BAO_KV/.peak" | tail -1)
|
||||
if [ "$peak" -gt 8 ]; then fail "$peak reads in flight, wanted at most 8"; fi
|
||||
if [ "$peak" -lt 2 ]; then fail "$peak read in flight, the reads did not overlap"; fi
|
||||
|
||||
if [ "$failures" != 0 ]; then
|
||||
echo "$failures of $count cases failed" >&2
|
||||
exit 1
|
||||
|
|
|
|||
Loading…
Reference in a new issue