diff --git a/docs/swarm/credentials.md b/docs/swarm/credentials.md index cfee085e..a1aaf9b7 100644 --- a/docs/swarm/credentials.md +++ b/docs/swarm/credentials.md @@ -174,7 +174,6 @@ single attempt, not for the retries: an attempt that fails ends that start, so the dependent starts with whatever is already on disk. When a later attempt lands a changed value, the homeserver, Grafana and collector readers restart the service that loaded it (`nix/host-modules/lib/refresh-consumer.nix`). -When the fetch unit can't read the credential file's mtime or the service's start time, it logs `cannot compare the mtime of … — leaving as it is` at `err` in its own journal, and the service keeps running on the old value until the next timer tick. ## Progressive enhancement diff --git a/nix/checks.nix b/nix/checks.nix index 6551916c..bc04772b 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -55,9 +55,6 @@ in inherit pkgs self nixosSystem; inherit (pkgs) lib; }; - # Runs ./host-modules/lib/refresh-consumer.nix's restart decision under a - # comma-decimal locale, against a stub `systemctl`. - script-test-refresh-consumer = import ./script-tests/refresh-consumer.nix { inherit pkgs; }; module-eval-nats-tls = import ./module-eval/nats-tls.nix { inherit pkgs self nixosSystem; inherit (pkgs) lib; diff --git a/nix/host-modules/lib/refresh-consumer.nix b/nix/host-modules/lib/refresh-consumer.nix index 8f74567e..c27923df 100644 --- a/nix/host-modules/lib/refresh-consumer.nix +++ b/nix/host-modules/lib/refresh-consumer.nix @@ -31,43 +31,27 @@ [ ! -f "$1" ] || [ "$(< "$1")" != "$2" ] } - # [systemctl option…]. Returns 0 when was last - # written after last entered `active` (or never has), 1 when - # it was not, and 2, logged at `err`, when either time does not read. Both - # are bare microsecond counts, with no decimal separator for a locale to - # change. A caller acts only on 0: the mtime stays, so the first run that - # can read both still acts, where acting on doubt acts on every run. The - # options are passed to `systemctl`, which is how a caller reaches into a - # container. - written_after_start() { + # [systemctl option…]. Nothing when last entered + # `active` after was last written. Otherwise a running is + # restarted and a failed one — it may have hit its start limit without the + # credential — is reset and started; one stopped on purpose stays stopped. + # The options are passed to every `systemctl` call, which is how + # `refresh_consumer` reaches into a container. `--no-block` because a + # caller ordered `Before=` its consumer must not wait on a job that waits + # on the caller (see `swarm-services-cert`'s propagation in ../hive-tls.nix). + refresh_unit() { local unit="$1" path="$2" started started_us=0 written_us shift 2 # Empty for a unit that has never been active, which `date` would # otherwise read as today's midnight. started="$(systemctl "$@" show --timestamp=us+utc -p ActiveEnterTimestamp --value "$unit")" if [ -n "$started" ]; then - started_us="$(date -u -d "$started" +%s%6N)" || started_us= - fi - written_us="$(date -u -r "$path" +%s%6N)" || written_us= - if [[ ! "$started_us" =~ ^[0-9]+$ || ! "$written_us" =~ ^[0-9]+$ ]]; then - echo "<3>cannot compare the mtime of $path ('$written_us') with the last start of $unit ('$started') — leaving $unit as it is" >&2 - return 2 + started_us="$(date -u -d "$started" +%s%6N)" fi + written_us="$(stat -c %.6Y "$path" | tr -d .)" if [ "$written_us" -le "$started_us" ]; then - return 1 + return 0 fi - } - - # [systemctl option…]. Nothing unless `written_after_start`. - # Otherwise a running is restarted and a failed one — it may have - # hit its start limit without the credential — is reset and started; one - # stopped on purpose stays stopped. `--no-block` because a caller ordered - # `Before=` its consumer must not wait on a job that waits on the caller - # (see `swarm-services-cert`'s propagation in ../hive-tls.nix). - refresh_unit() { - local unit="$1" path="$2" - shift 2 - written_after_start "$unit" "$path" "$@" || return 0 if systemctl "$@" is-failed --quiet "$unit"; then echo "$path changed after $unit last started, and $unit had failed — starting it" systemctl "$@" reset-failed "$unit" diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index db792c8b..5fd0df35 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -3429,7 +3429,6 @@ in echo "wrote ${agentCaCachePath}" fi - ${refreshConsumer} ${composeListenerBundle} compose_listener_bundle @@ -3438,12 +3437,11 @@ in echo "openbao in $machine is not running; it reads ${listenerClientCaPath} when it starts" exit 0 fi - newer=0 - written_after_start openbao.service ${listenerClientCaPath} --machine="$machine" || newer=$? - if [ "$newer" = 1 ]; then + started="$(systemctl --machine="$machine" show --timestamp=us+utc -p ActiveEnterTimestamp --value openbao.service)" + started_us="$(date -u -d "$started" +%s%6N)" + written_us="$(stat -c %.6Y ${listenerClientCaPath} | tr -d .)" + if [ "$written_us" -le "$started_us" ]; then echo "openbao started after ${listenerClientCaPath} last changed; nothing to pick up" - fi - if [ "$newer" != 0 ]; then exit 0 fi diff --git a/nix/module-eval/bao-basics.nix b/nix/module-eval/bao-basics.nix index c4614e67..a1d74de4 100644 --- a/nix/module-eval/bao-basics.nix +++ b/nix/module-eval/bao-basics.nix @@ -269,26 +269,16 @@ let { # openbao reads its client-CA file only at start, so picking up the # agent CA is a restart: automatic where the store unseals itself, - # printed where a human has to. The shamir arm is the control. Both - # decide on the shared locale-independent compare - # (../host-modules/lib/refresh-consumer.nix, run by - # `script-test-refresh-consumer`) and act only on its 0. + # printed where a human has to. The shamir arm is the control. name = "the agent PKI unit restarts openbao under pkcs11 and only prints the step under shamir"; ok = let restart = ''systemctl --machine="$machine" restart openbao.service''; - compare = '' - written_after_start openbao.service ${tlsDir}/listener-client-ca.pem --machine="$machine" || newer=$? - ''; p = baoPkcs11.systemd.services.swarm-bao-agent-pki.script; sh = baoShamir.systemd.services.swarm-bao-agent-pki.script; in lib.hasInfix restart p - && lib.all (s: lib.hasInfix compare s && lib.hasInfix ''if [ "$newer" != 0 ]; then'' s) [ - p - sh - ] - && !(lib.hasInfix "%.6Y" p) + && lib.hasInfix ''if [ "$written_us" -le "$started_us" ]; then'' p && !(lib.hasInfix restart sh) && lib.hasInfix "# then unseal" sh; } diff --git a/nix/script-tests/refresh-consumer.nix b/nix/script-tests/refresh-consumer.nix deleted file mode 100644 index 601ced46..00000000 --- a/nix/script-tests/refresh-consumer.nix +++ /dev/null @@ -1,19 +0,0 @@ -# `checks.script-test-refresh-consumer` — runs ../host-modules/lib/refresh-consumer.nix's -# `written_after_start`, `refresh_unit` and `refresh_consumer` against a stub -# `systemctl`, under a locale whose decimal separator is a comma. The cases -# are in ./refresh-consumer.sh. -# -# What runs is the rendered helper, sourced as its callers' scripts include it. -{ - pkgs, - ... -}: -pkgs.runCommand "hyperhive-script-test-refresh-consumer" - { - HELPER = pkgs.writeText "refresh-consumer.sh" (import ../host-modules/lib/refresh-consumer.nix { }); - LOCALE_ARCHIVE = "${pkgs.glibcLocales}/lib/locale/locale-archive"; - } - '' - ${pkgs.bash}/bin/bash ${./refresh-consumer.sh} - touch "$out" - '' diff --git a/nix/script-tests/refresh-consumer.sh b/nix/script-tests/refresh-consumer.sh deleted file mode 100644 index 55b55293..00000000 --- a/nix/script-tests/refresh-consumer.sh +++ /dev/null @@ -1,153 +0,0 @@ -# Cases for ./refresh-consumer.nix. Each case sets the credential file's mtime -# and the consumer's last start, runs the helper with a stub `systemctl` that -# records every call it is asked to act on, and compares that record and what -# the helper wrote to stderr. -set -uo pipefail - -failures=0 -count=0 - -fail() { - echo "FAILED: $case: $*" >&2 - failures=$((failures + 1)) -} - -dir=$(mktemp -d) -mkdir "$dir/bin" -calls=$dir/calls -path=$dir/token -: >"$path" - -# `show` prints $STARTED as systemd's `--timestamp=us+utc` does; `is-active` -# and `is-failed` answer from $ACTIVE and $FAILED; every other verb is -# appended to $calls. -cat >"$dir/bin/systemctl" <>"$calls" ;; -esac -STUB -chmod +x "$dir/bin/systemctl" -export PATH="$dir/bin:$PATH" ACTIVE=1 FAILED=0 STARTED= - -export LC_ALL=de_DE.UTF-8 - -# shellcheck source=/dev/null -. "$HELPER" - -# The locale the cases run under has to print the comma the helper must not -# depend on, or none of them tests it. -case="setup" -touch -d @1791232888.193159 "$path" -got=$(stat -c %.6Y "$path") -if [ "$got" != "1791232888,193159" ]; then fail "stat under $LC_ALL printed '$got', wanted 1791232888,193159"; fi - -# systemd's `--timestamp=us+utc` rendering of .. -systemd_ts() { - LC_ALL=C date -u -d "@$1" '+%a %Y-%m-%d %H:%M:%S.%6N UTC' -} - -# check -# -check() { - case=$1 - local want_calls=$2 want_err=$3 got err - shift 3 - count=$((count + 1)) - : >"$calls" - # `set -e` as the callers' NixOS `script`s run: a failed read must not end the - # caller. Not under `||` or `if`, which would switch `set -e` off. - local rc - ( - set -e - "$@" >/dev/null 2>"$dir/err" - ) - rc=$? - if [ "$rc" != 0 ]; then fail "exited $rc"; fi - err=$(cat "$dir/err") - got=$(cat "$calls") - if [ "$got" != "$want_calls" ]; then - fail "systemctl was asked:"$'\n'"$got"$'\n'"wanted:"$'\n'"$want_calls" - fi - # A failed read also leaves `date`'s own message ahead of the helper's line. - if [ -n "$want_err" ]; then err=$(printf '%s\n' "$err" | tail -n 1); fi - if [ "$err" != "$want_err" ]; then - fail "stderr:"$'\n'"$err"$'\n'"wanted:"$'\n'"$want_err" - fi -} - -touch -d @1791232888.193159 "$path" - -STARTED=$(systemd_ts 1791232900.000001) -check "a file older than the start restarts nothing" "" "" \ - refresh_unit gitea-runner-swarm.service "$path" -check "nor through a container" "" "" \ - refresh_consumer swarm-ci-runner gitea-runner-swarm.service "$path" - -STARTED=$(systemd_ts 1791232888.193159) -check "a file written in the start's microsecond restarts nothing" "" "" \ - refresh_unit gitea-runner-swarm.service "$path" - -STARTED=$(systemd_ts 1791232888.193158) -check "a file one microsecond newer than the start restarts the unit" \ - "try-restart --no-block gitea-runner-swarm.service" "" \ - refresh_unit gitea-runner-swarm.service "$path" -check "through a container, with --machine" \ - "--machine=swarm-ci-runner try-restart --no-block gitea-runner-swarm.service" "" \ - refresh_consumer swarm-ci-runner gitea-runner-swarm.service "$path" - -FAILED=1 -check "a failed unit is reset and started" \ - "reset-failed gitea-runner-swarm.service"$'\n'"start --no-block gitea-runner-swarm.service" "" \ - refresh_unit gitea-runner-swarm.service "$path" -FAILED=0 - -STARTED= -check "a unit never active is restarted" \ - "try-restart --no-block gitea-runner-swarm.service" "" \ - refresh_unit gitea-runner-swarm.service "$path" - -ACTIVE=0 -check "nothing while the container is down" "" "" \ - refresh_consumer swarm-ci-runner gitea-runner-swarm.service "$path" -ACTIVE=1 - -STARTED="not a time" -check "an unreadable start restarts nothing and says so at err" "" \ - "<3>cannot compare the mtime of $path ('1791232888193159') with the last start of gitea-runner-swarm.service ('not a time') — leaving gitea-runner-swarm.service as it is" \ - refresh_unit gitea-runner-swarm.service "$path" - -STARTED=$(systemd_ts 1791232900.000001) -check "a missing file restarts nothing and says so at err" "" \ - "<3>cannot compare the mtime of $dir/missing ('') with the last start of gitea-runner-swarm.service ('$STARTED') — leaving gitea-runner-swarm.service as it is" \ - refresh_unit gitea-runner-swarm.service "$dir/missing" - -# compare . `written_after_start` as -# `swarm-bao-agent-pki` calls it, which acts only on 0. -compare() { - case=$1 - count=$((count + 1)) - local rc - ( - set -e - written_after_start openbao.service "$3" --machine=swarm-bao 2>/dev/null - ) - rc=$? - if [ "$rc" != "$2" ]; then fail "returned $rc, wanted $2"; fi -} - -STARTED=$(systemd_ts 1791232900.000001) -compare "the compare is 1 for a file older than the start" 1 "$path" -STARTED=$(systemd_ts 1791232888.193158) -compare "the compare is 0 for a file newer than the start" 0 "$path" -STARTED="not a time" -compare "the compare is 2 for an unreadable start" 2 "$path" -STARTED=$(systemd_ts 1791232900.000001) -compare "the compare is 2 for a missing file" 2 "$dir/missing" - -echo "$count cases, $failures failed" -[ "$failures" = 0 ]