From 204abb1d0e7f9dd886a5130f1e51b3b44aa7f170 Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 30 Oct 2026 16:08:35 +0100 Subject: [PATCH] read grafana's datasource key from the store into /run, seeding an empty store from the old file --- docs/swarm/credentials-reference.md | 2 +- nix/checks.nix | 4 + nix/host-modules/lib/grafana-secret-key.nix | 58 ++++++++++++++ nix/host-modules/swarm-grafana.nix | 79 +++--------------- nix/module-eval/grafana.nix | 5 +- nix/script-tests/grafana-secret-key.nix | 68 ++++++++++++++++ nix/script-tests/grafana-secret-key.sh | 89 +++++++++++++++++++++ 7 files changed, 236 insertions(+), 69 deletions(-) create mode 100644 nix/host-modules/lib/grafana-secret-key.nix create mode 100644 nix/script-tests/grafana-secret-key.nix create mode 100644 nix/script-tests/grafana-secret-key.sh diff --git a/docs/swarm/credentials-reference.md b/docs/swarm/credentials-reference.md index 03cac2f9..b696ab7e 100644 --- a/docs/swarm/credentials-reference.md +++ b/docs/swarm/credentials-reference.md @@ -120,7 +120,7 @@ pieces fit, and the reasoning behind them, is in [`credentials.md`](credentials. | Grafana's copy of its OIDC secret | `swarm-bao-grafana-oidc.service` reads it out of the swarm secret store, **on every host that runs Grafana** | `/run/swarm-grafana-secrets/.secret` on the host's tmpfs, bind-mounted read-only into the `swarm-grafana` container — never on persistent disk, so a reboot fetches it again | same unit, same path — one route, co-located or not. Nothing for an operator to place beyond this host's store leaf, see [hive-level secrets](credentials.md#hive-level-secrets) | | Grafana's admin password | `swarm-bao-grafana-admin.service` creates it in the swarm secret store at `swarm/services//admin/password` when the store holds none, then copies it out, **on every host that runs Grafana**. Grafana's leaf may create that leaf but not update it, so the store keeps the first value, and the secret publisher can't write it. To rotate, run `bao kv metadata delete secret/swarm/services//admin/password` with a token that may, such as the root token (Grafana's, the publisher's and the controller's can't), then `systemctl restart swarm-bao-grafana-admin`; a plain `bao kv delete` isn't enough, see [the callout xkey's section](credentials.md#the-queues-callout-xkey) | `/run/swarm-grafana-secrets/admin-password` on the host's tmpfs, bind-mounted read-only into the `swarm-grafana` container, never on persistent disk: `security.admin_password` for a new database, and `grafana cli admin reset-admin-password` in `grafana.service`'s `preStart` for an existing one, on every start | same unit, same path. After a reboot the unit fetches the file again, and Grafana doesn't start until it exists, so it stays down rather than falling back to the upstream default `admin` | | Grafana's metrics bearer | `swarm-bao-grafana-metrics-bearer.service` creates it in the swarm secret store at `swarm/services//metrics/bearer` when the store holds none, then copies it out, **on every host that runs Grafana**. Same grant as the admin password, create without update. To rotate, run `bao kv metadata delete secret/swarm/services//metrics/bearer` with a token that may, then `systemctl restart swarm-bao-grafana-metrics-bearer` | `/run/swarm-grafana-secrets/metrics-bearer` on the host's tmpfs, bind-mounted read-only into the `swarm-grafana` container, holding `Bearer `: the metrics datasource's `Authorization` header. The logs datasource sends the same header. `swarm-metrics-auth` on the metrics store's host, and a second instance on the log store's host (`swarm-logs-grafana-auth`), read the stored value themselves, each under its own leaf | same unit, same path. The unit restarts Grafana when the file changes | -| Grafana's datasource encryption key | `swarm-bao-grafana-secret-key.service` stores it in the swarm secret store at `swarm/services//secret-key` when the store holds none, then copies it out, **on every host that runs Grafana**. It stores the key an earlier release generated at `/var/lib/grafana-secret/secret_key` in the container, so the key doesn't change, and mints a new one only on a host with no such file. Same grant as the admin password, create without update. The earlier file stays on disk. While the store's policy doesn't grant the leaf yet, or the store is down, the unit runs Grafana on that file | `/run/swarm-grafana-secrets/secret-key` on the host's tmpfs, bind-mounted read-only into the `swarm-grafana` container: `security.secret_key` | same unit, same path. Grafana doesn't start until the file exists. Changing the key leaves Grafana unable to read the datasource secrets it stored, so never delete the store path to rotate it | +| Grafana's datasource encryption key | `swarm-bao-grafana-secret-key.service` stores it in the swarm secret store at `swarm/services//secret-key` when the store holds none, then copies it out, **on every host that runs Grafana**. It stores the key an earlier release generated at `/var/lib/grafana-secret/secret_key` in the container, so the key doesn't change, and mints a new one only on a host with no such file. Same grant as the admin password, create without update. The earlier file stays on disk, unused once the store holds the key. A store that's down or doesn't grant the leaf fails the unit, and Grafana doesn't start | `/run/swarm-grafana-secrets/secret-key` on the host's tmpfs, bind-mounted read-only into the `swarm-grafana` container: `security.secret_key` | same unit, same path. Grafana doesn't start until the file exists. Changing the key leaves Grafana unable to read the datasource secrets it stored, so never delete the store path to rotate it | | the swarm controller's webhook secret | `swarm-controller` stores it in the swarm secret store at `swarm/controller/swarm-controller/forge/webhook-secret` when the store holds none: the value of `webhook-secret` in its state directory, or a new one when there is no such file. The state directory file stays on disk and is the fallback while the store is unreachable or its policy doesn't grant the path yet | the swarm secret store; the controller holds it in memory and registers the same value with the forge's hooks | nothing to place: the controller's store leaf already exists. A store that stays unreachable on a start with no state directory file leaves the webhook endpoint answering 503 | | the metrics store responder's own store leaf (cert + key) | `signLeaf` in `glue-bao-tls.nix` (`swarm-bao-pki.service`), on the host that runs the secret store, once, subject `deploy.bao.metricsAuthCommonName`. `swarm-bao-metrics-auth-policy.service` writes the `swarm-metrics-auth` role and its read grants on the bearer leaf and on the metrics store forwarder's client secret, nothing else | `/var/lib/swarm-bao-pki/metrics-auth.pem` and `metrics-auth-key.pem` on the store's host, handed to `swarm-metrics-auth` through `LoadCredential` (paths in argv, never values) and read by path by `swarm-victoriametrics-forwarder-oidc` | **the operator** places a leaf with that subject on the metrics store's host and names it in `deploy.victoriametrics.authBaoClientCertFile` / `authBaoClientKeyFile`; evaluation fails naming both until then. If the store is on another host, deploy that host first: the role must exist before the responder's login succeeds, and until it does the gateway refuses every Grafana query (fail closed) | | the secret publisher's store leaf (cert + key) | `signLeaf` in `glue-bao-tls.nix` (`swarm-bao-pki.service`), on the host that runs the secret store, once, subject `deploy.bao.secretPublisherCommonName`. `swarm-bao-secret-publisher-policy.service` writes the `swarm-secret-publisher` role: create and update under `swarm/hives/*`, `swarm/services/*` and the controller's client, and read on the `oidc/client` leaves it mints, the controller's client and authelia's five keys alone | `/var/lib/swarm-bao-pki/secret-publisher.pem` and `secret-publisher-key.pem` on the host, handed into authelia's container as nspawn system credentials, which `authelia-swarm-secrets`, `swarm-authelia-bridge` and the forwarder's `swarm-journal-forwarder-oidc` load with `LoadCredential`. nspawn won't start authelia's container while a named file is missing | **the operator** copies the leaf to authelia's host, beside the hive's own store leaf, where `glue-secret-publisher-bao-identity.nix` looks, or names it in `deploy.swarm-secret-publisher.baoClientCertFile` / `baoClientKeyFile` | diff --git a/nix/checks.nix b/nix/checks.nix index a35ed7b2..9ff80ff8 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -93,6 +93,10 @@ in # what is minted, seeded from a leftover file or left alone when the store # cannot answer. script-test-oidc-client-secrets = import ./script-tests/oidc-client-secrets.nix { inherit pkgs; }; + # Runs ./host-modules/lib/grafana-secret-key.nix's decision against a stub + # `bao`: a store that holds a key wins, an empty one takes the seed, and an + # unreachable one mints nothing. + script-test-grafana-secret-key = import ./script-tests/grafana-secret-key.nix { inherit pkgs; }; module-eval-nats-tls = import ./module-eval/nats-tls.nix { inherit pkgs self nixosSystem; inherit (pkgs) lib; diff --git a/nix/host-modules/lib/grafana-secret-key.nix b/nix/host-modules/lib/grafana-secret-key.nix new file mode 100644 index 00000000..fffcac83 --- /dev/null +++ b/nix/host-modules/lib/grafana-secret-key.nix @@ -0,0 +1,58 @@ +# The store's value for Grafana's datasource-encryption key, seeded once from +# the file an earlier release generated in the container. +# +# Grafana cannot read datasource secrets encrypted with another key, so a store +# with no value takes the file's, and a new key is minted only when there is no +# file either. The store is written with `-cas=0`, so a value it holds is never +# replaced and a lost race reads the winner's back. The file stays, unused. +# +# `grafana_secret_key` sets `GRAFANA_KEY` and returns 0, or returns 1 when the +# store does not log in, answer or take a value; nothing is minted then. +# +# Pure function — NOT a NixOS module. Call it from a unit's `script`, with +# `bao` and `coreutils` on its `path` and the `BAO_*` client environment set: +# +# script = '' +# ${import ./lib/grafana-secret-key.nix { }} +# grafana_secret_key secret/swarm/services/foo/secret-key /path/to/seed openssl rand -hex 32 || exit 1 +# ''; +{ }: +'' + # + grafana_secret_key() { + local leaf="$1" seed="$2" candidate err + shift 2 + GRAFANA_KEY="" + err="$(mktemp)" + + if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then + cat "$err" >&2 + rm -f "$err" + echo "could not log in to the swarm secret store." >&2 + return 1 + fi + export BAO_TOKEN + + if ! GRAFANA_KEY="$(bao kv get -field=value "$leaf" 2>"$err")" || [ -z "$GRAFANA_KEY" ]; then + if [ -s "$seed" ]; then + candidate="$(tr -d '\n' < "$seed")" + echo "storing the key at $seed in $leaf." + else + candidate="$("$@")" + echo "no earlier key; minting one in $leaf." + fi + if ! printf '%s' "$candidate" | bao kv put -cas=0 "$leaf" value=- >/dev/null 2>"$err"; then + cat "$err" >&2 + echo "could not create $leaf; reading it back in case another run created it." >&2 + fi + if ! GRAFANA_KEY="$(bao kv get -field=value "$leaf" 2>"$err")" || [ -z "$GRAFANA_KEY" ]; then + cat "$err" >&2 + rm -f "$err" + echo "the store did not return $leaf." >&2 + GRAFANA_KEY="" + return 1 + fi + fi + rm -f "$err" + } +'' diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index eb2bbbaf..d5a7c7e4 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -369,8 +369,8 @@ let # Grafana unable to read its stored datasource secrets. secretKeyStorePath = "secret/swarm/services/${cfg.oidc.clientId}/secret-key"; secretKeyPath = "${secretDir}/secret-key"; - # The earlier file as the host sees it. Never removed: it is the seed, and - # the key Grafana runs on while the store's policy does not grant the leaf. + # The earlier file as the host sees it: the seed for an empty store. It stays + # on disk, unused once the store holds the key. legacySecretKeyPath = "/var/lib/nixos-containers/${cfg.machine}/var/lib/grafana-secret/secret_key"; # The built-in `admin` account's password. Grafana's API takes basic auth @@ -411,6 +411,7 @@ let atomicWriteSecret = import ./lib/atomic-write-secret.nix { }; refreshConsumer = import ./lib/refresh-consumer.nix { }; + grafanaSecretKey = import ./lib/grafana-secret-key.nix { }; # First statement of each delivery script, ahead of the store login: the # directory is the container's bind source, so it has to exist even when the @@ -1035,12 +1036,11 @@ in # first, and a new key only when there is no earlier file either. Same # shape and grant as the admin password above. # - # The key never changes. An earlier release generated it in the container - # on first boot and kept it, so a host that has run Grafana stores that - # same key. While the store's policy does not grant the leaf yet (the - # store's host is deployed after this one) or the store is unreachable, - # the earlier file is what Grafana runs on, and the unit succeeds; a host - # without one has no key to keep, so there the unit fails and retries. + # The key never changes. An earlier release generated it in the container on + # first boot, so a host that has run Grafana stores that same key + # (`grafana_secret_key`, ./lib/grafana-secret-key.nix). The earlier file is + # only a seed: with the store down or not granting the leaf the unit fails + # and retries, and Grafana does not start without the key in /run. systemd.services.swarm-bao-grafana-secret-key = { description = "create Grafana's datasource encryption key in the swarm secret store if absent and fetch it"; # Ordered after the store where it runs here: @@ -1079,70 +1079,17 @@ in ${ensureSecretDir} - err="$(mktemp)" - trap 'rm -f "$err"' EXIT + ${grafanaSecretKey} - # The earlier file's key, or nothing. - earlier_key() { - if [ -s ${lib.escapeShellArg legacySecretKeyPath} ]; then - tr -d '\n' < ${lib.escapeShellArg legacySecretKeyPath} - fi - } + grafana_secret_key ${lib.escapeShellArg secretKeyStorePath} ${lib.escapeShellArg legacySecretKeyPath} openssl rand -hex 32 || exit 1 # Owner and mode as for the OIDC secret above, for its reasons. # `$__file{}` is read as Grafana starts, so a changed file needs a # restart. - land() { - if secret_differs ${lib.escapeShellArg secretKeyPath} "$1"; then - atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg secretKeyPath} "$1" - fi - refresh_consumer ${lib.escapeShellArg cfg.machine} grafana.service ${lib.escapeShellArg secretKeyPath} - } - - # The store cannot answer: run on the earlier file's key, or fail. - use_earlier_key() { - local key - key="$(earlier_key)" - if [ -z "$key" ]; then - echo "$1, and there is no earlier key at ${legacySecretKeyPath}; Grafana does not start without one." >&2 - exit 1 - fi - echo "$1; running Grafana on the key at ${legacySecretKeyPath}." >&2 - land "$key" - exit 0 - } - - if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then - cat "$err" >&2 - use_earlier_key "could not log in to swarm-bao with Grafana's certificate" + if secret_differs ${lib.escapeShellArg secretKeyPath} "$GRAFANA_KEY"; then + atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg secretKeyPath} "$GRAFANA_KEY" fi - export BAO_TOKEN - - # Read, create on a miss, read again: the admin password's sequence - # above, for its reasons, with the earlier key as the first candidate. - if ! key="$(bao kv get -field=value ${lib.escapeShellArg secretKeyStorePath} 2>"$err")"; then - candidate="$(earlier_key)" - if [ -n "$candidate" ]; then - echo "storing the key at ${legacySecretKeyPath} in ${secretKeyStorePath}." - else - candidate="$(openssl rand -hex 32)" - echo "no earlier key; minting one in ${secretKeyStorePath}." - fi - if ! printf '%s' "$candidate" | bao kv put -cas=0 ${lib.escapeShellArg secretKeyStorePath} value=- >/dev/null 2>"$err"; then - echo "could not create ${secretKeyStorePath}; reading it back in case another run created it." >&2 - cat "$err" >&2 - fi - if ! key="$(bao kv get -field=value ${lib.escapeShellArg secretKeyStorePath} 2>"$err")"; then - cat "$err" >&2 - use_earlier_key "swarm-bao did not return ${secretKeyStorePath}" - fi - fi - if [ -z "$key" ]; then - echo "swarm-bao returned an empty ${secretKeyStorePath}; leaving the file as it is." >&2 - exit 1 - fi - - land "$key" + refresh_consumer ${lib.escapeShellArg cfg.machine} grafana.service ${lib.escapeShellArg secretKeyPath} ''; }; diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix index 783df8bf..7f766cf4 100644 --- a/nix/module-eval/grafana.nix +++ b/nix/module-eval/grafana.nix @@ -268,8 +268,9 @@ let guest = m.containers.swarm-grafana.config; earlier = "/var/lib/nixos-containers/swarm-grafana/var/lib/grafana-secret/secret_key"; in - lib.hasInfix "bao kv put -cas=0 secret/swarm/services/swarm-grafana/secret-key value=-" unit.script - && lib.hasInfix "tr -d '\\n' < ${earlier}" unit.script + lib.hasInfix "bao kv put -cas=0 \"\$leaf\" value=-" unit.script + && lib.hasInfix "grafana_secret_key secret/swarm/services/swarm-grafana/secret-key ${earlier} openssl rand -hex 32" unit.script + && lib.length (lib.splitString earlier unit.script) == 2 && lib.hasInfix "atomic_write_secret 0400 196:0 /run/swarm-grafana-secrets/secret-key" unit.script && lib.elem "container@swarm-grafana.service" unit.before && unit.environment.BAO_CLIENT_CERT == "/etc/pki/bao-grafana-oidc.pem" diff --git a/nix/script-tests/grafana-secret-key.nix b/nix/script-tests/grafana-secret-key.nix new file mode 100644 index 00000000..c94c538c --- /dev/null +++ b/nix/script-tests/grafana-secret-key.nix @@ -0,0 +1,68 @@ +# `checks.script-test-grafana-secret-key` — runs +# ../host-modules/lib/grafana-secret-key.nix's `grafana_secret_key` against a +# stub `bao` that keeps its leaves in files. The cases are in +# ./grafana-secret-key.sh. +# +# What runs is the rendered helper, sourced as the unit's script includes it. +{ + pkgs, + ... +}: +let + # `login` answers from `FAKE_BAO_LOGIN_RC`. `kv get` and `kv put -cas=0` act + # on a file per leaf under `FAKE_BAO_KV`, and answer as bao does: a missing + # leaf is "No value found at " on stderr, status 2. `FAKE_BAO_DENY_READ` + # and `FAKE_BAO_DENY_WRITE` answer 403 as a policy that grants neither does. + # `FAKE_BAO_RACE` has another run create the leaf with that value between + # this one's read and its write. Every call is logged without the value + # `put` reads from stdin. + fakeBao = pkgs.writeShellScriptBin "bao" '' + echo "$*" >> "$FAKE_BAO_LOG" + case "$1 $2" in + "login -method=cert") + [ "$FAKE_BAO_LOGIN_RC" = 0 ] || { echo "fake bao: certificate refused" >&2; exit 2; } + printf 's.fake' + ;; + "kv get") + [ "''${BAO_TOKEN-}" = s.fake ] || { echo "fake bao: kv without the login's token" >&2; exit 97; } + [ "$3" = -field=value ] || { echo "fake bao: get without -field=value" >&2; exit 95; } + leaf="$FAKE_BAO_KV/$(printf '%s' "$4" | tr / _)" + [ -z "''${FAKE_BAO_DENY_READ-}" ] || { echo "fake bao: permission denied" >&2; exit 2; } + if [ -n "''${FAKE_BAO_RACE-}" ] && [ ! -e "$FAKE_BAO_KV/.raced" ]; then + touch "$FAKE_BAO_KV/.raced" + printf '%s' "$FAKE_BAO_RACE" > "$leaf" + echo "No value found at secret/data/x" >&2 + exit 2 + fi + [ -e "$leaf" ] || { echo "No value found at secret/data/x" >&2; exit 2; } + cat "$leaf" + echo + ;; + "kv put") + [ "''${BAO_TOKEN-}" = s.fake ] || { echo "fake bao: kv without the login's token" >&2; exit 97; } + [ "$3" = -cas=0 ] || { echo "fake bao: put without -cas=0" >&2; exit 96; } + [ "$5" = value=- ] || { echo "fake bao: put with the value as an argument" >&2; exit 94; } + leaf="$FAKE_BAO_KV/$(printf '%s' "$4" | tr / _)" + [ -z "''${FAKE_BAO_DENY_WRITE-}" ] || { echo "fake bao: permission denied" >&2; exit 2; } + [ ! -e "$leaf" ] || { echo "fake bao: check-and-set parameter did not match" >&2; exit 2; } + cat > "$leaf" + ;; + *) + echo "fake bao: unexpected call: $*" >&2 + exit 98 + ;; + esac + ''; +in +pkgs.runCommand "hyperhive-script-test-grafana-secret-key" + { + HELPER = pkgs.writeText "grafana-secret-key.sh" ( + import ../host-modules/lib/grafana-secret-key.nix { } + ); + FAKE_BAO_BIN = "${fakeBao}/bin"; + nativeBuildInputs = [ pkgs.coreutils ]; + } + '' + ${pkgs.bash}/bin/bash ${./grafana-secret-key.sh} + touch "$out" + '' diff --git a/nix/script-tests/grafana-secret-key.sh b/nix/script-tests/grafana-secret-key.sh new file mode 100644 index 00000000..061d679a --- /dev/null +++ b/nix/script-tests/grafana-secret-key.sh @@ -0,0 +1,89 @@ +# Cases for ./grafana-secret-key.nix. Each case sets up the store's leaf, the +# seed file and the stub's behaviour, runs `grafana_secret_key` the way the +# unit's script does, and compares its status, the key it hands over and what +# the store holds afterwards. +set -uo pipefail + +failures=0 +leaf=secret/swarm/services/t/secret-key + +# setup +setup() { + case=$1 + dir=$(mktemp -d) + mkdir "$dir/kv" + export FAKE_BAO_KV=$dir/kv FAKE_BAO_LOG=$dir/bao.log FAKE_BAO_LOGIN_RC=0 + unset FAKE_BAO_DENY_READ FAKE_BAO_DENY_WRITE FAKE_BAO_RACE + : >"$FAKE_BAO_LOG" + seed=$dir/secret_key + stored=$FAKE_BAO_KV/$(printf '%s' "$leaf" | tr / _) +} + +mint() { + printf 'minted-key\n' +} + +# expect +expect() { + ( + PATH="$FAKE_BAO_BIN:$PATH" + unset BAO_TOKEN + # shellcheck source=/dev/null + . "$HELPER" + rc=0 + grafana_secret_key "$leaf" "$seed" mint >/dev/null 2>"$dir/err" || rc=$? + printf '%s' "$rc" >"$dir/rc" + printf '%s' "$GRAFANA_KEY" >"$dir/key" + ) + held=$(cat "$stored" 2>/dev/null) + if [ "$(cat "$dir/rc")" != "$1" ]; then fail "exited $(cat "$dir/rc"), wanted $1: $(cat "$dir/err")"; fi + if [ "$(cat "$dir/key")" != "$2" ]; then fail "handed over '$(cat "$dir/key")', wanted '$2'"; fi + if [ "$held" != "$3" ]; then fail "the store holds '$held', wanted '$3'"; fi +} + +fail() { + echo "FAILED: $case: $*" >&2 + failures=$((failures + 1)) +} + +setup "an empty store takes the seed's key" +printf 'seed-key\n' >"$seed" +expect 0 seed-key seed-key +grep -q '^kv put -cas=0 ' "$FAKE_BAO_LOG" || fail "no create-only write" +[ -e "$seed" ] || fail "the seed file was removed" + +setup "a store that holds a key wins over the seed, and is not written" +printf 'seed-key\n' >"$seed" +printf 'store-key' >"$stored" +expect 0 store-key store-key +! grep -q '^kv put' "$FAKE_BAO_LOG" || fail "wrote to the store" + +setup "with no seed and an empty store, a key is minted and stored" +expect 0 minted-key minted-key + +setup "a store that refuses the login mints nothing, with or without a seed" +export FAKE_BAO_LOGIN_RC=1 +expect 1 "" "" +printf 'seed-key\n' >"$seed" +expect 1 "" "" + +setup "a store that refuses the read stores and hands over nothing" +printf 'seed-key\n' >"$seed" +export FAKE_BAO_DENY_READ=1 +expect 1 "" seed-key + +setup "a store that refuses the write hands over nothing" +export FAKE_BAO_DENY_WRITE=1 +expect 1 "" "" + +setup "a store that holds an empty value hands over nothing" +: >"$stored" +expect 1 "" "" + +setup "a lost race takes the winner's key" +printf 'seed-key\n' >"$seed" +export FAKE_BAO_RACE=other-key +expect 0 other-key other-key + +echo "$failures failures" +[ "$failures" = 0 ]