swarm-bao-agent-pki: decide openbao's restart on the shared locale-independent compare

`swarm-bao-agent-pki` carried its own copy of the mtime-vs-start compare,
with the same `stat -c %.6Y | tr -d .` read: on a comma-locale host the
`-le` errored and, under `seal = "pkcs11"`, openbao was restarted on
every run of the unit.

The compare moves out of `refresh_unit` into `written_after_start`
(0 newer, 1 not newer, 2 unreadable and logged at `err`), and both
`refresh_unit` and the agent-pki unit call it. The agent-pki unit cannot
call `refresh_unit` itself: it restarts with `restart`, not
`try-restart`, runs a fail-closed check of the bundle between the
compare and the restart, and under shamir prints the step instead of
restarting. It acts only on 0, so an unreadable time skips the restart
as `refresh_unit` does.

`script-test-refresh-consumer` covers the three return codes in the
agent-pki call shape; `module-eval-bao-basics` asserts both seal arms
decide on that call and that no `%.6Y` read is left in the unit.

Refs #5230

nix/host-modules/lib/refresh-consumer.nix

@@ -31,15 +31,15 @@
     [ ! -f "$1" ] || [ "$(< "$1")" != "$2" ]
   }
 
-  # <unit> <path> [systemctl option…]. Nothing when <unit> last entered
-  # `active` after <path> was last written. Otherwise a running <unit> is
-  # restarted and a failed one — it may have hit its start limit without the
-  # credential — is reset and started; one stopped on purpose stays stopped.
-  # The options are passed to every `systemctl` call, which is how
-  # `refresh_consumer` reaches into a container. `--no-block` because a
-  # caller ordered `Before=` its consumer must not wait on a job that waits
-  # on the caller (see `swarm-services-cert`'s propagation in ../hive-tls.nix).
-  refresh_unit() {
+  # <unit> <path> [systemctl option…]. Returns 0 when <path> was last
+  # written after <unit> last entered `active` (or <unit> never has), 1 when
+  # it was not, and 2, logged at `err`, when either time does not read. Both
+  # are bare microsecond counts, with no decimal separator for a locale to
+  # change. A caller acts only on 0: the mtime stays, so the first run that
+  # can read both still acts, where acting on doubt acts on every run. The
+  # options are passed to `systemctl`, which is how a caller reaches into a
+  # container.
+  written_after_start() {
     local unit="$1" path="$2" started started_us=0 written_us
     shift 2
     # Empty for a unit that has never been active, which `date` would
@@ -49,17 +49,25 @@
       started_us="$(date -u -d "$started" +%s%6N)" || started_us=
     fi
     written_us="$(date -u -r "$path" +%s%6N)" || written_us=
-    # Both are bare microsecond counts, with no decimal separator for a
-    # locale to change. One that is not is logged at `err` and nothing is
-    # restarted: the mtime stays, so the first run that can read both still
-    # restarts <unit>, where restarting on doubt restarts it on every run.
     if [[ ! "$started_us" =~ ^[0-9]+$ || ! "$written_us" =~ ^[0-9]+$ ]]; then
       echo "<3>cannot compare the mtime of $path ('$written_us') with the last start of $unit ('$started') — leaving $unit as it is" >&2
-      return 0
+      return 2
     fi
     if [ "$written_us" -le "$started_us" ]; then
-      return 0
+      return 1
     fi
+  }
+
+  # <unit> <path> [systemctl option…]. Nothing unless `written_after_start`.
+  # Otherwise a running <unit> is restarted and a failed one — it may have
+  # hit its start limit without the credential — is reset and started; one
+  # stopped on purpose stays stopped. `--no-block` because a caller ordered
+  # `Before=` its consumer must not wait on a job that waits on the caller
+  # (see `swarm-services-cert`'s propagation in ../hive-tls.nix).
+  refresh_unit() {
+    local unit="$1" path="$2"
+    shift 2
+    written_after_start "$unit" "$path" "$@" || return 0
     if systemctl "$@" is-failed --quiet "$unit"; then
       echo "$path changed after $unit last started, and $unit had failed — starting it"
       systemctl "$@" reset-failed "$unit"

nix/host-modules/swarm-bao.nix

@@ -3429,6 +3429,7 @@ in
             echo "wrote ${agentCaCachePath}"
           fi
 
+          ${refreshConsumer}
           ${composeListenerBundle}
           compose_listener_bundle
 
@@ -3437,11 +3438,12 @@ in
             echo "openbao in $machine is not running; it reads ${listenerClientCaPath} when it starts"
             exit 0
           fi
-          started="$(systemctl --machine="$machine" show --timestamp=us+utc -p ActiveEnterTimestamp --value openbao.service)"
-          started_us="$(date -u -d "$started" +%s%6N)"
-          written_us="$(stat -c %.6Y ${listenerClientCaPath} | tr -d .)"
-          if [ "$written_us" -le "$started_us" ]; then
+          newer=0
+          written_after_start openbao.service ${listenerClientCaPath} --machine="$machine" || newer=$?
+          if [ "$newer" = 1 ]; then
             echo "openbao started after ${listenerClientCaPath} last changed; nothing to pick up"
+          fi
+          if [ "$newer" != 0 ]; then
             exit 0
           fi
 

nix/module-eval/bao-basics.nix

@@ -269,16 +269,26 @@ let
     {
       # openbao reads its client-CA file only at start, so picking up the
       # agent CA is a restart: automatic where the store unseals itself,
-      # printed where a human has to. The shamir arm is the control.
+      # printed where a human has to. The shamir arm is the control. Both
+      # decide on the shared locale-independent compare
+      # (../host-modules/lib/refresh-consumer.nix, run by
+      # `script-test-refresh-consumer`) and act only on its 0.
       name = "the agent PKI unit restarts openbao under pkcs11 and only prints the step under shamir";
       ok =
         let
           restart = ''systemctl --machine="$machine" restart openbao.service'';
+          compare = ''
+            written_after_start openbao.service ${tlsDir}/listener-client-ca.pem --machine="$machine" || newer=$?
+          '';
           p = baoPkcs11.systemd.services.swarm-bao-agent-pki.script;
           sh = baoShamir.systemd.services.swarm-bao-agent-pki.script;
         in
         lib.hasInfix restart p
-        && lib.hasInfix ''if [ "$written_us" -le "$started_us" ]; then'' p
+        && lib.all (s: lib.hasInfix compare s && lib.hasInfix ''if [ "$newer" != 0 ]; then'' s) [
+          p
+          sh
+        ]
+        && !(lib.hasInfix "%.6Y" p)
         && !(lib.hasInfix restart sh)
         && lib.hasInfix "# then unseal" sh;
     }

nix/script-tests/refresh-consumer.nix

@@ -1,7 +1,7 @@
 # `checks.script-test-refresh-consumer` — runs ../host-modules/lib/refresh-consumer.nix's
-# `refresh_unit` and `refresh_consumer` against a stub `systemctl`, under a
-# locale whose decimal separator is a comma. The cases are in
-# ./refresh-consumer.sh.
+# `written_after_start`, `refresh_unit` and `refresh_consumer` against a stub
+# `systemctl`, under a locale whose decimal separator is a comma. The cases
+# are in ./refresh-consumer.sh.
 #
 # What runs is the rendered helper, sourced as its callers' scripts include it.
 {

nix/script-tests/refresh-consumer.sh

@@ -126,5 +126,28 @@ check "a missing file restarts nothing and says so at err" "" \
   "<3>cannot compare the mtime of $dir/missing ('') with the last start of gitea-runner-swarm.service ('$STARTED') — leaving gitea-runner-swarm.service as it is" \
   refresh_unit gitea-runner-swarm.service "$dir/missing"
 
+# compare <description> <want status> <path>. `written_after_start` as
+# `swarm-bao-agent-pki` calls it, which acts only on 0.
+compare() {
+  case=$1
+  count=$((count + 1))
+  local rc
+  (
+    set -e
+    written_after_start openbao.service "$3" --machine=swarm-bao 2>/dev/null
+  )
+  rc=$?
+  if [ "$rc" != "$2" ]; then fail "returned $rc, wanted $2"; fi
+}
+
+STARTED=$(systemd_ts 1791232900.000001)
+compare "the compare is 1 for a file older than the start" 1 "$path"
+STARTED=$(systemd_ts 1791232888.193158)
+compare "the compare is 0 for a file newer than the start" 0 "$path"
+STARTED="not a time"
+compare "the compare is 2 for an unreadable start" 2 "$path"
+STARTED=$(systemd_ts 1791232900.000001)
+compare "the compare is 2 for a missing file" 2 "$dir/missing"
+
 echo "$count cases, $failures failed"
 [ "$failures" = 0 ]