vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

drop the one-time migration: seed the store from a leftover file once, leave the file

Browse source 
A leftover plaintext client secret is stored once with -cas=0 when the store holds none, so nothing rotates. A store value wins over a different file. The file is no longer read back, compared or removed, and the migration unit is gone.
This commit is contained in:
atlas 2026-10-30 16:12:37 +01:00 • committed by mara
commit fcc9c8d0ba
6 changed files with 114 additions and 350 deletions
Download patch file Download diff file Expand all files Collapse all files

6
nix/checks.nix
View file

@ -88,9 +88,9 @@ in
# Runs ./host-modules/lib/store-seeded-secret.nix's seed, store-wins and mint
# decisions against a stub `bao` that keeps its leaves in files.
script-test-store-seeded-secret = import ./script-tests/store-seeded-secret.nix { inherit pkgs; };
# Runs ./host-modules/lib/oidc-client-secrets.nix's mint and migration
# decisions for authelia's client secrets against stub `bao` and `authelia`:
# what is minted, stored and removed, and what is left alone when the store
# Runs ./host-modules/lib/oidc-client-secrets.nix's mint decisions for
# authelia's client secrets against stub `bao` and `authelia`:
# what is minted, seeded from a leftover file or left alone when the store
# cannot answer.
script-test-oidc-client-secrets = import ./script-tests/oidc-client-secrets.nix { inherit pkgs; };
module-eval-nats-tls = import ./module-eval/nats-tls.nix {

68
nix/host-modules/lib/oidc-client-secrets.nix
View file

@ -4,21 +4,15 @@
# the store answers.
#
# `mint_client_secret` keeps every client with a digest. A client without one
# gets it from the secret that exists: the plaintext file an earlier release
# left, else the store's value. Only a client with none of the three is minted,
# gets it from the store's secret, which an earlier release's plaintext file
# seeds once when the store holds none. Only a client with neither is minted,
# and then the store's leaf is created (`-cas=0`, so an existing value is never
# replaced) before the digest is written, so a digest on disk always has a
# secret behind it. A store that cannot be read never mints, because a second
# secret would rotate the first.
#
# `migrate_client_secret` moves the plaintext file an earlier release left in
# `$clients/<id>.secret` into the store and removes it once a read-back shows
# the store holds the same value. It never overwrites a store value and never
# removes a file the store does not match.
#
# Return statuses: 0 done, 1 retry later (the store could not answer, or a
# digest could not be computed), 3 (migration only) the file and the store hold
# different secrets, and only an operator can say which is right.
# digest could not be computed).
#
# Pure function — NOT a NixOS module. Call it after `./store-seeded-secret.nix`
# and `./atomic-write-secret.nix`, with `store_init` and `store_login` run and
@ -72,14 +66,20 @@
mint_client_secret() {
local id="$1" leaf="$2" dig="$clients/$1.digest" legacy="$clients/$1.secret"
local state=0 out secret digest
if [ -s "$legacy" ]; then
if [ -s "$dig" ]; then
return 0
fi
client_secret_derive_digest "$id" "$(< "$legacy")" "$legacy"
return
fi
client_secret_read "$leaf" || state=$?
if [ "$state" = 1 ] && [ -s "$legacy" ]; then
# A file an earlier release left: its value goes into the store once, so
# the secret keeps its value. The file stays, unused.
printf '%s' "$(< "$legacy")" | store_bao kv put -cas=0 "$leaf" value=- >/dev/null 2>"$STORE_ERR" \
|| cat "$STORE_ERR" >&2
state=0
client_secret_read "$leaf" || state=$?
if [ "$state" = 1 ]; then
echo "oidc client $id: could not store the secret from $legacy in $leaf; not minting" >&2
return 1
fi
echo "oidc client $id: stored the secret from $legacy in the store"
fi
if [ "$state" = 0 ]; then
if [ -s "$dig" ]; then
chmod 0600 "$dig"
@ -119,40 +119,4 @@
MINTED=1
echo "minted client secret for $id into the store"
}
# <client id> <leaf>
migrate_client_secret() {
local id="$1" leaf="$2" legacy="$clients/$1.secret" value state=0
[ -e "$legacy" ] || return 0
value="$(< "$legacy")"
if [ -z "$value" ]; then
echo "<3>oidc client $id: $legacy is empty; keeping it" >&2
return 3
fi
client_secret_read "$leaf" || state=$?
if [ "$state" = 2 ]; then
echo "oidc client $id: the store cannot be read; keeping $legacy" >&2
return 1
fi
if [ "$state" = 0 ]; then
if [ "$CLIENT_SECRET_VALUE" != "$value" ]; then
echo "<3>oidc client $id: the store at $leaf and $legacy hold different secrets; keeping both" >&2
return 3
fi
rm -f "$legacy" || return 1
echo "oidc client $id: already in the store, removed the file $legacy"
return 0
fi
if ! printf '%s' "$value" | store_bao kv put -cas=0 "$leaf" value=- >/dev/null 2>"$STORE_ERR"; then
cat "$STORE_ERR" >&2
echo "oidc client $id: could not create $leaf; reading it back in case another run created it" >&2
fi
client_secret_read "$leaf" || return 1
if [ "$CLIENT_SECRET_VALUE" != "$value" ]; then
echo "<3>oidc client $id: the store at $leaf and $legacy hold different secrets; keeping both" >&2
return 3
fi
rm -f "$legacy" || return 1
echo "oidc client $id: stored in the store, removed the file $legacy"
}
''

87
nix/host-modules/swarm-authelia.nix
View file

@ -237,7 +237,6 @@ let
publisherCfg.enable
&& publisherCfg.baoClientCertFile != null
&& publisherCfg.baoClientKeyFile != null;
migrateUnit = "${unitName}-client-secrets";
forwarderFetchUnit = "swarm-journal-forwarder-oidc";
# The publisher's store identity, handed into the container as system
# credentials (credential id → host source). The forwarder's fetch presents
@ -580,9 +579,7 @@ let
storeRetry = import ./lib/store-retry.nix { };
# The bridge's own client secret, in the memory-backed runtime directory of
# its unit. While a plaintext file from an earlier release exists the bridge
# keeps using it, since it is the value the digest was made from; the file
# goes once the store holds the same value, and the store's takes over.
# its unit, fetched from the store before each start.
bridgeSecretFile = "/run/swarm-authelia-bridge/client-secret";
bridgeSecretFetch = ''
set -euo pipefail
@ -591,12 +588,9 @@ let
${storeSeededSecret}
${oidcClientSecrets}
legacy=${lib.escapeShellArg "${clientsDir}/${bridgeClientId}.secret"}
store_init
store_login
if [ -s "$legacy" ]; then
value="$(< "$legacy")"
elif client_secret_read ${lib.escapeShellArg (clientLeaf bridgeClientId)}; then
if client_secret_read ${lib.escapeShellArg (clientLeaf bridgeClientId)}; then
value="$CLIENT_SECRET_VALUE"
else
echo "the store holds no readable secret for ${bridgeClientId}" >&2
@ -1094,7 +1088,6 @@ in
++ lib.optionals publishes [
"opentelemetry-collector.service"
"${forwarderFetchUnit}.service"
"${migrateUnit}.service"
"${unitName}-secrets.service"
"swarm-authelia-bridge.service"
];
@ -1133,79 +1126,6 @@ in
serviceConfig.LoadCredential = lib.attrNames storeFiles;
};
# Moves the plaintext files an earlier release left in `clientsDir`
# into the store and removes each once the store holds the same
# value. With no file left it does nothing, store or no store. Nothing
# orders authelia after it: authelia starts from its digests.
systemd.services.${migrateUnit} = lib.mkIf publishes {
description = "move authelia's plaintext OIDC client secrets into the swarm secret store";
wantedBy = [ "multi-user.target" ];
after = [ "${unitName}-secrets.service" ];
path = [
pkgs.coreutils
pkgs.gnugrep
deployCfg.bao.package
];
startLimitBurst = storeRetry.startLimitBurst;
startLimitIntervalSec = storeRetry.startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
User = unitName;
Group = socketGroup;
UMask = "0077";
LoadCredential = lib.attrNames storeFiles;
TimeoutStartSec = 120;
RestartPreventExitStatus = 3;
SyslogIdentifier = migrateUnit;
};
environment =
let
store = storeIdentityIn migrateUnit;
in
{
BAO_ADDR = store.addr;
BAO_CLIENT_CERT = store.clientCertFile;
BAO_CLIENT_KEY = store.clientKeyFile;
}
// lib.optionalAttrs (store.caFile != null) { BAO_CACERT = store.caFile; };
script = ''
set -uo pipefail
${storeSeededSecret}
${oidcClientSecrets}
clients=${lib.escapeShellArg clientsDir}
known=${lib.escapeShellArg (lib.concatMapStrings (c: " ${c.id}") secretClients + " ")}
pending=0
for f in "$clients"/*.secret; do
[ -e "$f" ] || continue
id="$(basename "$f" .secret)"
case "$known" in
*" $id "*) pending=1 ;;
*) echo "left $f in place: no client $id is registered here" >&2 ;;
esac
done
if [ "$pending" = 0 ]; then
echo "no plaintext client secret files to move"
exit 0
fi
store_init
store_login
status=0
${lib.concatMapStrings (c: ''
rc=0
migrate_client_secret ${lib.escapeShellArg c.id} ${lib.escapeShellArg (clientLeaf c.id)} || rc=$?
if [ "$rc" != 0 ] && [ "$status" != 1 ]; then
status="$rc"
fi
'') secretClients}
exit "$status"
'';
};
# The authelia binary itself, so an operator who gets a shell
# in here can run `authelia crypto hash generate` to make a
# password for the users file. Without it the container runs
@ -1323,8 +1243,7 @@ in
after = [
"${unitName}-secrets.service"
"${unitName}.service"
]
++ lib.optional publishes "${migrateUnit}.service";
];
wants = [
"${unitName}-secrets.service"
"${unitName}.service"

70
nix/module-eval/secret-publisher.nix
View file

@ -323,7 +323,6 @@ let
onStdin = s: puts s != [ ] && lib.all (l: lib.hasInfix "value=-" l) (puts s);
in
onStdin (inAuthelia secretPublisherHere).systemd.services.authelia-swarm-secrets.script
&& onStdin (inAuthelia secretPublisherHere).systemd.services.authelia-swarm-client-secrets.script
&& !(onStdin "printf '%s' \"$secret\" | bao kv put -cas=0 x value=\"$secret\"")
&& !(onStdin "# kv put x value=-\nbao kv put x value=@/tmp/f")
&& onStdin "printf '%s' \"$secret\" | bao kv put -cas=0 x value=-";
@ -369,8 +368,6 @@ let
&& !(lib.hasInfix "bao kv" (lib.head (lib.splitString "bao login" c)));
in
holdsTokenFirst (inAuthelia secretPublisherHere).systemd.services.authelia-swarm-secrets.script
&& holdsTokenFirst (inAuthelia secretPublisherHere)
.systemd.services.authelia-swarm-client-secrets.script
&& holdsTokenFirst (inAuthelia secretPublisherHere).systemd.services.swarm-authelia-bridge.preStart
&& holdsTokenFirst baoWithMatrix.systemd.services.swarm-bao-matrix-oidc.script
# Controls, so a clean verdict above means something. In order: a bare
@ -389,24 +386,24 @@ let
# would have left the unit rendering only on the one deployment that has
# no use for it. It runs beside the mint, in authelia's container, and
# never on the host.
name = "the client secret migration holding an identity runs in authelia's container on a host with no store";
name = "the mint holding an identity runs in authelia's container on a host with no store";
ok =
let
m = secretPublisherRemote;
c = inAuthelia m;
in
!m.services.hyperhive.deploy.bao.enable
&& (c.systemd.services ? authelia-swarm-client-secrets)
&& !(m.systemd.services ? authelia-swarm-client-secrets)
&& lib.hasInfix "mint_client_secret" c.systemd.services.authelia-swarm-secrets.script
&& !(m.systemd.services ? authelia-swarm-secrets)
# Control: the host's own unit set is the one read above.
&& (m.systemd.services ? "container@swarm-authelia");
}
{
# What makes the arm above able to fail. Without a certificate the
# plaintext has no store to go to, so the mint keeps it in a file beside
# the digest as before, the migration does not exist, the bridge reads
# the digest as before, the bridge reads
# that file, and nspawn gets no credential to load.
name = "an IdP host with no store identity keeps the client secrets in files and renders no migration";
name = "an IdP host with no store identity keeps the client secrets in files";
ok =
let
m = secretPublisherNoIdentity;
@ -414,7 +411,6 @@ let
mint = c.systemd.services.authelia-swarm-secrets.script;
in
m.services.hyperhive.deploy.authelia.enable
&& !(c.systemd.services ? authelia-swarm-client-secrets)
&& lib.hasInfix "mint() {" mint
&& !(lib.hasInfix "mint_client_secret" mint)
&&
@ -423,7 +419,6 @@ let
&& !(c.systemd.services.swarm-authelia-bridge.serviceConfig ? RuntimeDirectory)
&& !(lib.any (lib.hasPrefix "--load-credential=") m.containers.swarm-authelia.extraFlags)
# Controls: with an identity, the same lookups find the other shape.
&& ((inAuthelia secretPublisherHere).systemd.services ? authelia-swarm-client-secrets)
&& lib.hasInfix "mint_client_secret" (
(inAuthelia secretPublisherHere).systemd.services.authelia-swarm-secrets.script
)
@ -456,20 +451,16 @@ let
&& lib.hasInfix "client_secret_read ${leaf}" bridge.preStart
&& lib.hasInfix "mint_client_secret swarm-authelia-bridge ${leaf}" here.systemd.services.authelia-swarm-secrets.script
&& lib.elem "publisher-bao-cert" bridge.serviceConfig.LoadCredential
&& bridge.serviceConfig.Restart == "on-failure"
# The file it reads from is the one the earlier release left, until the
# migration moves it into the store.
&& lib.hasInfix "legacy=/var/lib/authelia-swarm/oidc-clients/swarm-authelia-bridge.secret" bridge.preStart;
&& bridge.serviceConfig.Restart == "on-failure";
}
{
# Every non-native client is migrated and minted at the leaf its reader
# Every non-native client is minted at the leaf its reader
# fetches, not only the ones with a reader in the publisher's list: the
# bridge's own, which only the bridge reads, is the one that list lacks.
name = "the migration visits every client at the leaf the mint creates, and a failure that no retry changes is not retried";
name = "the mint stores every client at the leaf its reader fetches";
ok =
let
c = inAuthelia secretPublisherHere;
migrate = c.systemd.services.authelia-swarm-client-secrets;
mint = c.systemd.services.authelia-swarm-secrets;
pairs = [
"hive-h1 secret/swarm/hives/h1/oidc/client"
@ -479,19 +470,11 @@ let
"swarm-authelia-bridge secret/swarm/services/swarm-authelia-bridge/oidc/client"
];
in
lib.all (
p:
lib.hasInfix "migrate_client_secret ${p}" migrate.script
&& lib.hasInfix "mint_client_secret ${p}" mint.script
) pairs
# A native client is public: it has no secret to mint or move.
&& !(lib.hasInfix "swarmctl" migrate.script)
lib.all (p: lib.hasInfix "mint_client_secret ${p}" mint.script) pairs
# A native client is public: it has no secret to mint.
&& !(lib.hasInfix "mint_client_secret swarmctl" mint.script)
&& migrate.serviceConfig.RestartPreventExitStatus == 3
# The mint answers authelia's password prompt through a pty.
&& lib.any (p: lib.hasPrefix "util-linux" (p.name or "")) mint.path
&& migrate.serviceConfig.Restart == "on-failure"
&& migrate.serviceConfig.Type == "oneshot";
&& lib.any (p: lib.hasPrefix "util-linux" (p.name or "")) mint.path;
}
{
# Every key authelia signs or encrypts with comes from the store, seeded
@ -532,8 +515,8 @@ let
name = "the store identity reaches authelia's container as system credentials";
ok =
let
dir = "/run/credentials/authelia-swarm-client-secrets.service";
unit = m: (inAuthelia m).systemd.services.authelia-swarm-client-secrets;
dir = "/run/credentials/authelia-swarm-secrets.service";
unit = m: (inAuthelia m).systemd.services.authelia-swarm-secrets;
flags = m: m.containers.swarm-authelia.extraFlags;
here = unit secretPublisherHere;
in
@ -566,33 +549,6 @@ let
)
&& (unit secretPublisherRemote).environment.BAO_CLIENT_CERT == "${dir}/publisher-bao-cert";
}
{
# Authelia is the swarm's IdP and has to come up while the store is down.
# Nothing it starts after or needs may be the migration, and the
# migration may not order itself in front of authelia.
name = "authelia does not wait for the client secret migration";
ok =
let
c = inAuthelia secretPublisherHere;
authelia = c.systemd.services.authelia-swarm;
publisher = c.systemd.services.authelia-swarm-client-secrets;
unit = "authelia-swarm-client-secrets.service";
in
!(lib.any (l: lib.elem unit (authelia.${l} or [ ])) [
"after"
"requires"
"wants"
"bindsTo"
"requisite"
])
&& !(lib.any (l: lib.elem "authelia-swarm.service" (publisher.${l} or [ ])) [
"before"
"requiredBy"
])
# Controls: both lookups find the real units.
&& authelia.serviceConfig ? ExecStart
&& lib.elem "authelia-swarm-secrets.service" publisher.after;
}
];
in
runGroup "secret-publisher" cases

3
nix/script-tests/oidc-client-secrets.nix
View file

@ -1,6 +1,5 @@
# `checks.script-test-oidc-client-secrets` — runs
# ../host-modules/lib/oidc-client-secrets.nix's `mint_client_secret` and
# `migrate_client_secret` against a stub `bao` that keeps its leaves in files
# ../host-modules/lib/oidc-client-secrets.nix's `mint_client_secret` against a stub `bao` that keeps its leaves in files
# and a stub `authelia` that prints a fixed password and digest. The cases are
# in ./oidc-client-secrets.sh.
#

142
nix/script-tests/oidc-client-secrets.sh
View file

@ -2,7 +2,7 @@
# digest and the plaintext file an earlier release left, and the stub's
# behaviour, runs the function under `set -e` as the units' scripts do, and
# compares its status, what the store holds, what is left on disk and whether
# `authelia` ran.
# `authelia` minted.
set -uo pipefail
failures=0
@ -172,27 +172,57 @@ if [ "$(stat -c %a "$digest")" != 600 ]; then fail "digest mode is $(stat -c %a
if ! logged "oidc client $id: wrote the digest from the secret in the store"; then fail "no info line"; fi
no_secret_in_argv
setup "mint: a plaintext file with its digest is left for the migration"
setup "mint: a plaintext file the store lacks seeds the store once, mints nothing and keeps the digest and the file"
printf 'old-secret\n' >"$legacy"
printf 'old-digest\n' >"$digest"
run mint_client_secret
expect 0
held old-secret
digest_is old-digest
if ran_authelia; then fail "minted although a file held the secret"; fi
if store_called; then fail "called the store"; fi
if [ "$(cat "$legacy")" != old-secret ]; then fail "changed or removed the file"; fi
if ! grep -q '^kv put -cas=0 ' "$FAKE_BAO_LOG"; then fail "no create-only write was made"; fi
no_secret_in_argv
setup "mint: a plaintext file with no digest gets the digest of the file's secret, and the file and the store are untouched"
setup "mint: a plaintext file with no digest seeds the store and gets the digest of the file's secret"
printf 'old-secret\n' >"$legacy"
run mint_client_secret
expect 0
held old-secret
digest_is digest-of-old-secret
held ""
if [ "$(cat "$legacy")" != old-secret ]; then fail "changed the file"; fi
if ran_authelia; then fail "minted over the file's secret"; fi
if store_called; then fail "called the store"; fi
if ! logged "oidc client $id: wrote the digest from the secret in $legacy"; then fail "no info line"; fi
no_secret_in_argv
setup "mint: the store's secret wins over a different file, silently, and nothing is written"
printf 'old-secret\n' >"$legacy"
printf 'other-secret' >"$stored"
printf 'old-digest\n' >"$digest"
run mint_client_secret
expect 0
held other-secret
digest_is old-digest
if grep -q '^kv put' "$FAKE_BAO_LOG"; then fail "wrote to a leaf that held a value"; fi
if ran_authelia; then fail "minted"; fi
setup "mint: a file with a store that refuses the login keeps the digest and mints nothing"
printf 'old-secret\n' >"$legacy"
printf 'old-digest\n' >"$digest"
export FAKE_BAO_LOGIN_RC=1
run mint_client_secret
expect 0
digest_is old-digest
if ran_authelia; then fail "minted with the store down"; fi
setup "mint: a file whose seed write is refused is a failure, and nothing is minted"
printf 'old-secret\n' >"$legacy"
printf 'old-digest\n' >"$digest"
export FAKE_BAO_DENY_WRITE=1
run mint_client_secret
expect 1
held ""
if ran_authelia; then fail "minted although the file held the secret"; fi
setup "mint: a secret whose digest cannot be computed is a failure and writes no digest"
printf 'old-secret' >"$stored"
export FAKE_AUTHELIA_NODIGEST=1
@ -232,104 +262,6 @@ if [ "$(id -u)" != 0 ]; then
digest_is ""
fi
# --- migrate_client_secret ------------------------------------------------
setup "migrate: a file the store lacks is stored, read back and removed"
printf 'old-secret\n' >"$legacy"
run migrate_client_secret
expect 0
held old-secret
if [ -e "$legacy" ]; then fail "left the file"; fi
if ! logged "stored in the store, removed the file $legacy"; then fail "no stored line"; fi
if ran_authelia; then fail "minted"; fi
no_secret_in_argv
setup "migrate: a file the store already holds is removed, and the store is not written"
printf 'old-secret\n' >"$legacy"
printf 'old-secret\n' >"$stored"
run migrate_client_secret
expect 0
held "old-secret"
if [ -e "$legacy" ]; then fail "left the file"; fi
if grep -q '^kv put' "$FAKE_BAO_LOG"; then fail "wrote to a leaf that held a value"; fi
if ! logged "already in the store, removed the file $legacy"; then fail "no removed line"; fi
setup "migrate: a store holding a different secret keeps both and reports an error"
printf 'old-secret\n' >"$legacy"
printf 'other-secret' >"$stored"
run migrate_client_secret
expect 3
held other-secret
if [ "$(cat "$legacy")" != old-secret ]; then fail "changed the file"; fi
if grep -q '^kv put' "$FAKE_BAO_LOG"; then fail "wrote to a leaf that held a value"; fi
if ! logged "<3>oidc client $id"; then fail "no err-priority line"; fi
setup "migrate: a store that refuses the login keeps the file and retries"
printf 'old-secret\n' >"$legacy"
export FAKE_BAO_LOGIN_RC=1
run migrate_client_secret
expect 1
if [ ! -e "$legacy" ]; then fail "removed the file"; fi
if store_called; then fail "called kv after a refused login"; fi
setup "migrate: a store that refuses the read keeps the file and retries"
printf 'old-secret\n' >"$legacy"
export FAKE_BAO_DENY_READ=1
run migrate_client_secret
expect 1
if [ ! -e "$legacy" ]; then fail "removed the file"; fi
if grep -q '^kv put' "$FAKE_BAO_LOG"; then fail "wrote although the read was refused"; fi
setup "migrate: a store that refuses the write keeps the file and retries"
printf 'old-secret\n' >"$legacy"
export FAKE_BAO_DENY_WRITE=1
run migrate_client_secret
expect 1
if [ ! -e "$legacy" ]; then fail "removed the file"; fi
held ""
setup "migrate: losing the race to a different secret keeps the file and reports an error"
printf 'old-secret\n' >"$legacy"
export FAKE_BAO_RACE=winner-secret
run migrate_client_secret
expect 3
held winner-secret
if [ ! -e "$legacy" ]; then fail "removed the file"; fi
setup "migrate: an empty file is kept and reported"
: >"$legacy"
run migrate_client_secret
expect 3
if [ ! -e "$legacy" ]; then fail "removed the file"; fi
held ""
setup "migrate: no file is nothing to do, store or no store"
export FAKE_BAO_LOGIN_RC=1
run migrate_client_secret
expect 0
if store_called; then fail "called the store"; fi
if [ "$(id -u)" != 0 ]; then
setup "migrate: a file that cannot be removed is a failure, not a move"
printf 'old-secret\n' >"$legacy"
chmod 500 "$clients"
run migrate_client_secret
chmod 700 "$clients"
expect 1
held old-secret
if ! [ -e "$legacy" ]; then fail "file vanished"; fi
if logged "removed the file"; then fail "logged a removal that did not happen"; fi
setup "migrate: a file the store already holds that cannot be removed is a failure"
printf 'old-secret\n' >"$legacy"
printf 'old-secret' >"$stored"
chmod 500 "$clients"
run migrate_client_secret
chmod 700 "$clients"
expect 1
if logged "removed the file"; then fail "logged a removal that did not happen"; fi
fi
if [ "$failures" != 0 ]; then
echo "$failures of $count cases failed" >&2
exit 1