diff --git a/nix/host-modules/lib/oidc-client-secrets.nix b/nix/host-modules/lib/oidc-client-secrets.nix index 4014aad9..ca4217c5 100644 --- a/nix/host-modules/lib/oidc-client-secrets.nix +++ b/nix/host-modules/lib/oidc-client-secrets.nix @@ -27,7 +27,7 @@ client_secret_read() { CLIENT_SECRET_VALUE="" [ "$STORE_UP" = 1 ] || return 2 - if CLIENT_SECRET_VALUE="$(store_bao kv get -field=value "$1" 2>"$STORE_ERR")" \ + if CLIENT_SECRET_VALUE="$(store_read "$1")" \ && [ -n "$CLIENT_SECRET_VALUE" ]; then return 0 fi diff --git a/nix/host-modules/lib/store-seeded-secret.nix b/nix/host-modules/lib/store-seeded-secret.nix index e03a99fc..691d4aae 100644 --- a/nix/host-modules/lib/store-seeded-secret.nix +++ b/nix/host-modules/lib/store-seeded-secret.nix @@ -8,6 +8,11 @@ # race reads the winner's back. A read of the leaf that fails for any reason # but "No value found" prints the store's error and neither mints nor writes. # +# `store_prefetch` reads many leaves concurrently, and the first `store_read` of +# a leaf returns that answer, status and error included, instead of asking +# again, so what a caller decides from a read is the same. Later reads of the +# leaf, such as one that checks a write, ask the store. +# # Pure function — NOT a NixOS module. Call it from a module's `let`: # # storeSeededSecret = import ./lib/store-seeded-secret.nix { }; @@ -31,7 +36,8 @@ store_init() { STORE_ERR="$(mktemp)" STORE_UP=0 - trap 'rm -f "$STORE_ERR"' EXIT + STORE_PREFETCH="$(mktemp -d)" + trap 'rm -rf "$STORE_ERR" "$STORE_PREFETCH"' EXIT } # Sets STORE_UP. A store that cannot be logged in to leaves every secret to @@ -48,6 +54,41 @@ fi } + # … Starts the first read of each leaf now, at most STORE_PREFETCH_JOBS + # at a time, and returns when all have ended. Each read is bounded by + # `store_bao`, and its answer is kept for `store_read`. Does nothing without + # a login. + store_prefetch() { + local leaf key + [ "$STORE_UP" = 1 ] || return 0 + for leaf in "$@"; do + while [ "$(jobs -rp | wc -l)" -ge "''${STORE_PREFETCH_JOBS:-8}" ]; do + wait -n || true + done + key="$STORE_PREFETCH/$(printf '%s' "$leaf" | tr / _)" + ( + rc=0 + store_bao kv get -field=value "$leaf" >"$key.out" 2>"$key.err" || rc=$? + echo "$rc" >"$key.rc" + ) & + done + wait + } + + # . `store_bao kv get -field=value `, with the error in + # $STORE_ERR: the answer `store_prefetch` kept, once, then the store's. + store_read() { + local key="$STORE_PREFETCH/$(printf '%s' "$1" | tr / _)" rc + if [ -e "$key.rc" ]; then + rc="$(< "$key.rc")" + cat "$key.out" + cat "$key.err" >"$STORE_ERR" + rm -f "$key.out" "$key.err" "$key.rc" + return "$rc" + fi + store_bao kv get -field=value "$1" 2>"$STORE_ERR" + } + # . The value on stdout; status 1 when # neither the store nor the file can give one. `kv get` exits 2 both for a # leaf that holds nothing ("No value found at ") and for a refused @@ -58,7 +99,7 @@ local leaf="$1" earlier="$2" value candidate rc=0 shift 2 if [ "$STORE_UP" = 1 ]; then - value="$(store_bao kv get -field=value "$leaf" 2>"$STORE_ERR")" || rc=$? + value="$(store_read "$leaf")" || rc=$? if [ "$rc" = 0 ] && [ -n "$value" ]; then printf '%s' "$value" return 0 diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index 5b28b4d9..14523aa4 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -526,12 +526,14 @@ let # cannot be read fails the run too, after the files are rendered from the # digests (`STORE_UNREADABLE`, checked at the end of the unit's script), so # the unit retries until the store answers and authelia has started from the - # digests meanwhile. + # digests meanwhile. Runs after `storeKeysScript`, which defines + # `phase_done` and has already fetched these leaves. storeMintScript = '' ${oidcClientSecrets} ${lib.concatMapStrings (c: '' mint_client_secret ${lib.escapeShellArg c.id} ${lib.escapeShellArg (clientLeaf c.id)} || { rc=$?; [ "$rc" = 2 ] || exit "$rc"; STORE_UNREADABLE=1; } '') secretClients} + phase_done clients ''; # The OIDC half of the first-boot generator, kept out of the script @@ -624,6 +626,12 @@ let done ''; + # Every leaf the unit reads, for `store_prefetch`: the keys and the client + # secrets. The reads that follow are answered from what it fetched. + storeLeaves = + map keyLeaf (randomKeys ++ lib.optional oidcEnabled "oidc-issuer") + ++ lib.optionals oidcEnabled (map (c: clientLeaf c.id) secretClients); + # The same keys from the store, at the same paths. The store's value wins, # then the file's, which seeds the store when it holds none, so a key never # changes value by moving; a new one is minted only where neither exists. @@ -632,8 +640,21 @@ let storeKeysScript = '' ${storeSeededSecret} ${refreshConsumer} + + # . One line: how long the phase that ends here took, so a slow store + # shows where the time went. Not `SECONDS`, which counts whole seconds. + PHASE_AT="$EPOCHREALTIME" + phase_done() { + local now="$EPOCHREALTIME" + echo "phase $1: $(( (''${now//[.,]/} - ''${PHASE_AT//[.,]/}) / 1000 )) ms" + PHASE_AT="$now" + } + store_init store_login + phase_done login + store_prefetch ${lib.concatMapStringsSep " " lib.escapeShellArg storeLeaves} + phase_done reads mint_hex() { head -c 64 /dev/urandom | od -An -tx1 | tr -d ' \n' @@ -661,6 +682,7 @@ let keep_key "$f" mint_hex done ${lib.optionalString oidcEnabled "keep_key oidc-issuer mint_issuer"} + phase_done keys ''; # A netns of its own on the hive bridge: the container dials the store, the @@ -1197,6 +1219,10 @@ in # accepts a connection and never answers: this unit is in the # boot wait of a container with a one-minute start limit. # What a killed run left is picked up by the next one. + # + # The run's store reads are concurrent (`store_prefetch`): its + # length is the login, a few rounds of reads and the writes, + # not one round trip per leaf. Raising this raises that wait. TimeoutStartSec = 30; RestartSec = 15; RestartSteps = 3; diff --git a/nix/module-eval/secret-publisher.nix b/nix/module-eval/secret-publisher.nix index ed5ffe92..d0c1d720 100644 --- a/nix/module-eval/secret-publisher.nix +++ b/nix/module-eval/secret-publisher.nix @@ -529,6 +529,56 @@ let && !(onDisk.environment ? BAO_ADDR) && !(onDisk.serviceConfig ? LoadCredential); } + { + # The unit runs inside the container's boot wait, so its start budget is + # the ceiling of that wait and stays at 30 s; what fits the budget is that + # every leaf the run reads is fetched together right after the login, then + # the sequential reads are answered from that. The leaves are those of + # the keys and of every `mint_client_secret` call, found by the same + # lookup, so a leaf added to either and left out of the prefetch fails + # here. The controls: the no-identity unit has no prefetch, and a call + # with its leaf left out would not be found by the lookup. + name = "the secrets unit fetches every leaf it reads in one concurrent pass after the login, inside its 30 s budget"; + ok = + let + unit = m: (inAuthelia m).systemd.services.authelia-swarm-secrets; + s = (unit secretPublisherHere).script; + lines = lib.splitString "\n" s; + calls = lib.filter (l: lib.hasPrefix "store_prefetch secret/" l) lines; + prefetch = lib.head calls; + leafOf = l: builtins.match "mint_client_secret [^ ]+ ([^ ]+) .*" l; + clientLeaves = lib.concatMap ( + l: + let + m = leafOf l; + in + lib.optional (m != null) (lib.head m) + ) lines; + hasLeaf = leaf: lib.elem leaf (lib.splitString " " prefetch); + phases = lib.filter (lib.hasPrefix "phase_done ") lines; + in + (unit secretPublisherHere).serviceConfig.TimeoutStartSec == 30 + && lib.length calls == 1 + && lib.length clientLeaves >= 4 + && lib.all hasLeaf clientLeaves + && lib.all (k: hasLeaf "secret/swarm/services/swarm-authelia/keys/${k}") [ + "jwt" + "session" + "storage-encryption" + "oidc-hmac" + "oidc-issuer" + ] + && lib.length (lib.splitString " " prefetch) == lib.length clientLeaves + 6 + && + phases == [ + "phase_done login" + "phase_done reads" + "phase_done keys" + "phase_done clients" + ] + && !(hasLeaf "secret/swarm/services/not-a-leaf") + && !(lib.hasInfix "\nstore_prefetch secret/" (unit secretPublisherNoIdentity).script); + } { # The leaf crosses into the container as nspawn system credentials, and # the unit loads them and names them in its environment. On the store's diff --git a/nix/script-tests/oidc-client-secrets.sh b/nix/script-tests/oidc-client-secrets.sh index ae344665..d0d83959 100644 --- a/nix/script-tests/oidc-client-secrets.sh +++ b/nix/script-tests/oidc-client-secrets.sh @@ -24,7 +24,7 @@ setup() { mkdir "$dir/kv" "$dir/clients" export FAKE_BAO_KV=$dir/kv FAKE_BAO_LOG=$dir/bao.log FAKE_BAO_LOGIN_RC=0 export FAKE_AUTHELIA_LOG=$dir/authelia.log - unset FAKE_BAO_DENY_READ FAKE_BAO_DENY_WRITE FAKE_BAO_RACE FAKE_AUTHELIA_EMPTY FAKE_AUTHELIA_NODIGEST + unset FAKE_BAO_DENY_READ FAKE_BAO_DENY_WRITE FAKE_BAO_RACE FAKE_AUTHELIA_EMPTY FAKE_AUTHELIA_NODIGEST PREFETCH : >"$FAKE_BAO_LOG" : >"$FAKE_AUTHELIA_LOG" clients=$dir/clients @@ -34,7 +34,8 @@ setup() { } # run — the function for $id and $leaf, with the stubs first on -# PATH, after the store login the units make first. Sets $rc and $minted. +# PATH, after the store login the units make first and, when $PREFETCH is set, +# `store_prefetch` of the leaf. Sets $rc and $minted. run() { minted=$( set -e @@ -45,6 +46,7 @@ run() { clients=$dir/clients store_init store_login 2>>"$dir/err" + [ -z "${PREFETCH-}" ] || store_prefetch "$leaf" 2>>"$dir/err" rc=0 "$1" "$id" "$leaf" 2>>"$dir/err" >>"$dir/out" || rc=$? echo "${MINTED-}" @@ -262,6 +264,52 @@ if [ "$(id -u)" != 0 ]; then digest_is "" fi +# kv_gets: how many `kv get` calls the stub logged. +kv_gets() { + grep -c '^kv get' "$FAKE_BAO_LOG" || true +} + +setup "prefetch: a client the store and the disk both hold is left alone after one read" +PREFETCH=1 +printf 'old-secret' >"$stored" +printf 'old-digest\n' >"$digest" +run mint_client_secret +expect 0 +held old-secret +digest_is old-digest +if ran_authelia; then fail "minted although the store held a value"; fi +if [ "$(kv_gets)" != 1 ]; then fail "$(kv_gets) reads, wanted 1"; fi + +setup "prefetch: a digest whose secret is gone from the store is minted again, and the read back asks the store" +PREFETCH=1 +printf 'old-digest\n' >"$digest" +run mint_client_secret +expect 0 +held minted-secret +digest_is '$pbkdf2-sha512$fake' +if [ "$minted" != 1 ]; then fail "did not report the mint"; fi +if [ "$(kv_gets)" != 2 ]; then fail "$(kv_gets) reads, wanted the prefetch and the read back"; fi + +setup "prefetch: a store that refuses the read leaves the digest, mints nothing and asks to retry" +PREFETCH=1 +printf 'old-digest\n' >"$digest" +export FAKE_BAO_DENY_READ=1 +run mint_client_secret +expect 2 +digest_is old-digest +if ran_authelia; then fail "minted although the read was refused"; fi +if grep -q '^kv put' "$FAKE_BAO_LOG"; then fail "wrote although the read was refused"; fi +if ! logged "the store cannot be read; keeping its digest"; then fail "no keep line"; fi + +setup "prefetch: no digest but a secret in the store gets the digest of that secret" +PREFETCH=1 +printf 'old-secret' >"$stored" +run mint_client_secret +expect 0 +held old-secret +digest_is digest-of-old-secret +if ran_authelia; then fail "minted over the store's secret"; fi + if [ "$failures" != 0 ]; then echo "$failures of $count cases failed" >&2 exit 1 diff --git a/nix/script-tests/store-seeded-secret.nix b/nix/script-tests/store-seeded-secret.nix index 1f5c72dc..cf39b40f 100644 --- a/nix/script-tests/store-seeded-secret.nix +++ b/nix/script-tests/store-seeded-secret.nix @@ -16,7 +16,9 @@ let # `FAKE_BAO_READ_5XX` is a server error on the read. A leaf nobody wrote is # "No value found at ", status 2, as `bao kv get` answers it. # `FAKE_BAO_RACE` has another run create the leaf between this one's read and - # its write. Every call is logged without the value `put` reads from stdin. + # its write. `FAKE_BAO_READ_DELAY` holds each read that long and records, in + # `$FAKE_BAO_KV/.peak`, how many reads were in flight with it. Every call is + # logged without the value `put` reads from stdin. fakeBao = pkgs.writeShellScriptBin "bao" '' echo "$*" >> "$FAKE_BAO_LOG" case "$1 $2" in @@ -28,6 +30,12 @@ let [ "''${BAO_TOKEN-}" = s.fake ] || { echo "fake bao: kv without the login's token" >&2; exit 97; } leaf="$FAKE_BAO_KV/$(printf '%s' "$4" | tr / _)" [ -z "''${FAKE_BAO_READ_TIMEOUT-}" ] || exit 124 + if [ -n "''${FAKE_BAO_READ_DELAY-}" ]; then + touch "$FAKE_BAO_KV/.inflight.$$" + ls "$FAKE_BAO_KV"/.inflight.* | wc -l >> "$FAKE_BAO_KV/.peak" + sleep "$FAKE_BAO_READ_DELAY" + rm -f "$FAKE_BAO_KV/.inflight.$$" + fi [ -z "''${FAKE_BAO_READ_5XX-}" ] || { echo "Error reading secret/data/x: Error making API request. Code: 503. Errors: * Vault is sealed" >&2; exit 2; } [ -z "''${FAKE_BAO_DENY_READ-}" ] || { echo "fake bao: permission denied" >&2; exit 2; } if [ -n "''${FAKE_BAO_RACE-}" ] && [ ! -e "$FAKE_BAO_KV/.raced" ]; then diff --git a/nix/script-tests/store-seeded-secret.sh b/nix/script-tests/store-seeded-secret.sh index 35825fe0..3494300d 100644 --- a/nix/script-tests/store-seeded-secret.sh +++ b/nix/script-tests/store-seeded-secret.sh @@ -22,7 +22,7 @@ setup() { dir=$(mktemp -d) mkdir "$dir/kv" export FAKE_BAO_KV=$dir/kv FAKE_BAO_LOG=$dir/bao.log FAKE_BAO_LOGIN_RC=0 - unset FAKE_BAO_DENY_READ FAKE_BAO_DENY_WRITE FAKE_BAO_RACE FAKE_BAO_READ_TIMEOUT FAKE_BAO_READ_5XX + unset FAKE_BAO_DENY_READ FAKE_BAO_DENY_WRITE FAKE_BAO_RACE FAKE_BAO_READ_TIMEOUT FAKE_BAO_READ_5XX FAKE_BAO_READ_DELAY PREFETCH : >"$FAKE_BAO_LOG" earlier=$dir/earlier minted=$dir/minted @@ -34,7 +34,8 @@ mint() { printf 'minted-value\n' } -# run — `store_seeded` for $leaf and $earlier with the stub first on PATH. +# run — `store_seeded` for $leaf and $earlier with the stub first on PATH, after +# `store_prefetch` of the leaf when $PREFETCH is set. # Sets $rc and $got. run() { got=$( @@ -45,6 +46,7 @@ run() { . "$HELPER" store_init store_login 2>>"$dir/err" + [ -z "${PREFETCH-}" ] || store_prefetch "$leaf" 2>>"$dir/err" store_seeded "$leaf" "$earlier" mint 2>"$dir/err" ) rc=$? @@ -212,6 +214,90 @@ run expect 1 "" if ran_mint; then fail "minted with no store to hold the value"; fi +# kv_gets: how many `kv get` calls the stub logged. +kv_gets() { + grep -c '^kv get' "$FAKE_BAO_LOG" || true +} + +setup "prefetch: a leaf the store holds is read once, and the result is the same" +PREFETCH=1 +printf 'stored-value' >"$stored" +printf 'old-value\n' >"$earlier" +run +expect 0 stored-value +held stored-value +no_write +no_mint +if [ "$(kv_gets)" != 1 ]; then fail "$(kv_gets) reads, wanted 1"; fi + +setup "prefetch: a leaf the store lacks is seeded from the file as without it" +PREFETCH=1 +printf 'old-value\n' >"$earlier" +run +expect 0 old-value +held old-value +if ran_mint; then fail "minted although the file held a value"; fi + +setup "prefetch: a refused read uses the file's value, writes nothing and prints the refusal, as without it" +PREFETCH=1 +printf 'old-value\n' >"$earlier" +export FAKE_BAO_DENY_READ=1 FAKE_BAO_DENY_WRITE=1 +run +expect 0 old-value +held "" +no_write +no_mint +err_has "could not read $leaf from the swarm secret store" +err_has "permission denied" + +setup "prefetch: a read that times out is not treated as an absent leaf" +PREFETCH=1 +printf 'stored-value' >"$stored" +export FAKE_BAO_READ_TIMEOUT=1 +run +expect 1 "" +held stored-value +no_write +no_mint +err_has "(exit status 124)" + +setup "prefetch: a store that refuses the login reads nothing" +PREFETCH=1 +printf 'old-value\n' >"$earlier" +export FAKE_BAO_LOGIN_RC=1 +run +expect 0 old-value +if grep -q '^kv' "$FAKE_BAO_LOG"; then fail "called kv after a refused login"; fi + +setup "prefetch: many leaves are read together, at most 8 at once, each once, and a second read of one asks the store" +export FAKE_BAO_READ_DELAY=0.3 +many=() +for i in $(seq 1 20); do + many+=("secret/swarm/many/$i") + printf 'v%s' "$i" >"$FAKE_BAO_KV/$(printf '%s' "secret/swarm/many/$i" | tr / _)" +done +want=$(for i in $(seq 1 20); do printf 'v%s\n' "$i"; done) +got=$( + set -e + PATH="$FAKE_BAO_BIN:$PATH" + unset BAO_TOKEN + # shellcheck source=/dev/null + . "$HELPER" + store_init + store_login + store_prefetch "${many[@]}" + for l in "${many[@]}"; do + store_read "$l" + echo + done + store_read "${many[0]}" +) +if [ "$got" != "$want"$'\n'v1 ]; then fail "values differ: $got"; fi +if [ "$(kv_gets)" != 21 ]; then fail "$(kv_gets) reads, wanted 20 and the second read of one"; fi +peak=$(sort -n "$FAKE_BAO_KV/.peak" | tail -1) +if [ "$peak" -gt 8 ]; then fail "$peak reads in flight, wanted at most 8"; fi +if [ "$peak" -lt 2 ]; then fail "$peak read in flight, the reads did not overlap"; fi + if [ "$failures" != 0 ]; then echo "$failures of $count cases failed" >&2 exit 1