swarm-bao-agent-pki: decide openbao's restart on the shared locale-independent compare

`swarm-bao-agent-pki` carried its own copy of the mtime-vs-start compare,
with the same `stat -c %.6Y | tr -d .` read: on a comma-locale host the
`-le` errored and, under `seal = "pkcs11"`, openbao was restarted on
every run of the unit.

The compare moves out of `refresh_unit` into `written_after_start`
(0 newer, 1 not newer, 2 unreadable and logged at `err`), and both
`refresh_unit` and the agent-pki unit call it. The agent-pki unit cannot
call `refresh_unit` itself: it restarts with `restart`, not
`try-restart`, runs a fail-closed check of the bundle between the
compare and the restart, and under shamir prints the step instead of
restarting. It acts only on 0, so an unreadable time skips the restart
as `refresh_unit` does.

`script-test-refresh-consumer` covers the three return codes in the
agent-pki call shape; `module-eval-bao-basics` asserts both seal arms
decide on that call and that no `%.6Y` read is left in the unit.

Refs #5230

nix/host-modules/lib/refresh-consumer.nix

@@ -31,15 +31,15 @@
     [ ! -f "$1" ] || [ "$(< "$1")" != "$2" ]
   }
 
-  # <unit> <path> [systemctl option…]. Nothing when <unit> last entered
+  # <unit> <path> [systemctl option…]. Returns 0 when <path> was last
-  # `active` after <path> was last written. Otherwise a running <unit> is
+  # written after <unit> last entered `active` (or <unit> never has), 1 when
-  # restarted and a failed one — it may have hit its start limit without the
+  # it was not, and 2, logged at `err`, when either time does not read. Both
-  # credential — is reset and started; one stopped on purpose stays stopped.
+  # are bare microsecond counts, with no decimal separator for a locale to
-  # The options are passed to every `systemctl` call, which is how
+  # change. A caller acts only on 0: the mtime stays, so the first run that
-  # `refresh_consumer` reaches into a container. `--no-block` because a
+  # can read both still acts, where acting on doubt acts on every run. The
-  # caller ordered `Before=` its consumer must not wait on a job that waits
+  # options are passed to `systemctl`, which is how a caller reaches into a
-  # on the caller (see `swarm-services-cert`'s propagation in ../hive-tls.nix).
+  # container.
-  refresh_unit() {
+  written_after_start() {
     local unit="$1" path="$2" started started_us=0 written_us
     shift 2
     # Empty for a unit that has never been active, which `date` would
@@ -49,17 +49,25 @@
       started_us="$(date -u -d "$started" +%s%6N)" || started_us=
     fi
     written_us="$(date -u -r "$path" +%s%6N)" || written_us=
-    # Both are bare microsecond counts, with no decimal separator for a
-    # locale to change. One that is not is logged at `err` and nothing is
-    # restarted: the mtime stays, so the first run that can read both still
-    # restarts <unit>, where restarting on doubt restarts it on every run.
     if [[ ! "$started_us" =~ ^[0-9]+$ || ! "$written_us" =~ ^[0-9]+$ ]]; then
       echo "<3>cannot compare the mtime of $path ('$written_us') with the last start of $unit ('$started') — leaving $unit as it is" >&2
-      return 0
+      return 2
     fi
     if [ "$written_us" -le "$started_us" ]; then
-      return 0
+      return 1
     fi
+  }
+
+  # <unit> <path> [systemctl option…]. Nothing unless `written_after_start`.
+  # Otherwise a running <unit> is restarted and a failed one — it may have
+  # hit its start limit without the credential — is reset and started; one
+  # stopped on purpose stays stopped. `--no-block` because a caller ordered
+  # `Before=` its consumer must not wait on a job that waits on the caller
+  # (see `swarm-services-cert`'s propagation in ../hive-tls.nix).
+  refresh_unit() {
+    local unit="$1" path="$2"
+    shift 2
+    written_after_start "$unit" "$path" "$@" || return 0
     if systemctl "$@" is-failed --quiet "$unit"; then
       echo "$path changed after $unit last started, and $unit had failed — starting it"
       systemctl "$@" reset-failed "$unit"

nix/host-modules/swarm-bao.nix

@@ -3429,6 +3429,7 @@ in
             echo "wrote ${agentCaCachePath}"
           fi
 
+          ${refreshConsumer}
           ${composeListenerBundle}
           compose_listener_bundle
 
@@ -3437,11 +3438,12 @@ in
             echo "openbao in $machine is not running; it reads ${listenerClientCaPath} when it starts"
             exit 0
           fi
-          started="$(systemctl --machine="$machine" show --timestamp=us+utc -p ActiveEnterTimestamp --value openbao.service)"
+          newer=0
-          started_us="$(date -u -d "$started" +%s%6N)"
+          written_after_start openbao.service ${listenerClientCaPath} --machine="$machine" || newer=$?
-          written_us="$(stat -c %.6Y ${listenerClientCaPath} | tr -d .)"
+          if [ "$newer" = 1 ]; then
-          if [ "$written_us" -le "$started_us" ]; then
             echo "openbao started after ${listenerClientCaPath} last changed; nothing to pick up"
+          fi
+          if [ "$newer" != 0 ]; then
             exit 0
           fi
 

nix/module-eval/bao-basics.nix

@@ -269,16 +269,26 @@ let
     {
       # openbao reads its client-CA file only at start, so picking up the
       # agent CA is a restart: automatic where the store unseals itself,
-      # printed where a human has to. The shamir arm is the control.
+      # printed where a human has to. The shamir arm is the control. Both
+      # decide on the shared locale-independent compare
+      # (../host-modules/lib/refresh-consumer.nix, run by
+      # `script-test-refresh-consumer`) and act only on its 0.
       name = "the agent PKI unit restarts openbao under pkcs11 and only prints the step under shamir";
       ok =
         let
           restart = ''systemctl --machine="$machine" restart openbao.service'';
+          compare = ''
+            written_after_start openbao.service ${tlsDir}/listener-client-ca.pem --machine="$machine" || newer=$?
+          '';
           p = baoPkcs11.systemd.services.swarm-bao-agent-pki.script;
           sh = baoShamir.systemd.services.swarm-bao-agent-pki.script;
         in
         lib.hasInfix restart p
-        && lib.hasInfix ''if [ "$written_us" -le "$started_us" ]; then'' p
+        && lib.all (s: lib.hasInfix compare s && lib.hasInfix ''if [ "$newer" != 0 ]; then'' s) [
+          p
+          sh
+        ]
+        && !(lib.hasInfix "%.6Y" p)
         && !(lib.hasInfix restart sh)
         && lib.hasInfix "# then unseal" sh;
     }

nix/script-tests/refresh-consumer.nix

@@ -1,7 +1,7 @@
 # `checks.script-test-refresh-consumer` — runs ../host-modules/lib/refresh-consumer.nix's
-# `refresh_unit` and `refresh_consumer` against a stub `systemctl`, under a
+# `written_after_start`, `refresh_unit` and `refresh_consumer` against a stub
-# locale whose decimal separator is a comma. The cases are in
+# `systemctl`, under a locale whose decimal separator is a comma. The cases
-# ./refresh-consumer.sh.
+# are in ./refresh-consumer.sh.
 #
 # What runs is the rendered helper, sourced as its callers' scripts include it.
 {

nix/script-tests/refresh-consumer.sh

@@ -126,5 +126,28 @@ check "a missing file restarts nothing and says so at err" "" \
   "<3>cannot compare the mtime of $dir/missing ('') with the last start of gitea-runner-swarm.service ('$STARTED') — leaving gitea-runner-swarm.service as it is" \
   refresh_unit gitea-runner-swarm.service "$dir/missing"
 
+# compare <description> <want status> <path>. `written_after_start` as
+# `swarm-bao-agent-pki` calls it, which acts only on 0.
+compare() {
+  case=$1
+  count=$((count + 1))
+  local rc
+  (
+    set -e
+    written_after_start openbao.service "$3" --machine=swarm-bao 2>/dev/null
+  )
+  rc=$?
+  if [ "$rc" != "$2" ]; then fail "returned $rc, wanted $2"; fi
+}
+
+STARTED=$(systemd_ts 1791232900.000001)
+compare "the compare is 1 for a file older than the start" 1 "$path"
+STARTED=$(systemd_ts 1791232888.193158)
+compare "the compare is 0 for a file newer than the start" 0 "$path"
+STARTED="not a time"
+compare "the compare is 2 for an unreadable start" 2 "$path"
+STARTED=$(systemd_ts 1791232900.000001)
+compare "the compare is 2 for a missing file" 2 "$dir/missing"
+
 echo "$count cases, $failures failed"
 [ "$failures" = 0 ]