wait for grafana's socket and read the forge's user list whole in the mint

nix/host-modules/hive-forge/default.nix

@@ -71,6 +71,12 @@ let
 
   baoCfg = config.services.hyperhive.swarm.bao;
   storeRetry = import ../lib/store-retry.nix { };
+  retryInJob = import ../lib/retry-in-job.nix { };
+
+  # How long the controller-account mint re-checks that the forge lists the
+  # account it just ensured. The mint is in the forge container's boot wait,
+  # whose `container@` limit is a minute.
+  accountRecheckSec = 15;
   storeRefreshTimer = import ../lib/store-refresh-timer.nix { };
   atomicWriteSecret = import ../lib/atomic-write-secret.nix { };
   refreshConsumer = import ../lib/refresh-consumer.nix { };
@@ -1052,14 +1058,28 @@ in
               # an already-provisioned non-admin account to work around
               # that (every hive from before this fix has one) — she
               # promotes those by hand instead.
-              if ! forgejo admin user list | grep -qE "[[:space:]]${swarmControllerForgeUser}[[:space:]]"; then
+              ${retryInJob}
+
+              # The list is read whole before `grep` sees it: `grep -q` leaves
+              # at its first match, and under `pipefail` forgejo's write to the
+              # closed pipe would fail the pipeline for an account that is
+              # listed.
+              account_listed() {
+                local list
+                list="$(forgejo admin user list "$@")" || return 1
+                grep -qE "[[:space:]]${swarmControllerForgeUser}[[:space:]]" <<<"$list"
+              }
+
+              if ! account_listed; then
                 forgejo admin user create \
                   --username ${lib.escapeShellArg swarmControllerForgeUser} \
                   --email ${lib.escapeShellArg "${swarmControllerForgeUser}@hyperhive.local"} \
                   --admin --random-password --must-change-password=false
                 echo "created forge account ${swarmControllerForgeUser}"
               fi
-              if ! forgejo admin user list | grep -qE "[[:space:]]${swarmControllerForgeUser}[[:space:]]"; then
+              # This unit is part of the container's boot wait, so the re-check
+              # stays well under its start limit (`accountRecheckSec`).
+              if ! retry_in_job ${toString accountRecheckSec} account_listed; then
                 echo "forge account ${swarmControllerForgeUser} absent after creation" >&2
                 exit 1
               fi
@@ -1099,7 +1119,7 @@ in
                   # The revocation below goes through the forge's admin API,
                   # which refuses a token of an account that is not a site
                   # admin; each retry would then leave one more token live.
-                  if ! forgejo admin user list --admin | grep -qE "[[:space:]]${swarmControllerForgeUser}[[:space:]]"; then
+                  if ! account_listed --admin; then
                     echo "forge account ${swarmControllerForgeUser} is not a site admin; promote it by hand. No token minted." >&2
                     exit 1
                   fi

nix/host-modules/swarm-grafana.nix

@@ -427,6 +427,12 @@ let
   privateNetwork = true;
   storeRetry = import ./lib/store-retry.nix { };
   storeRefreshTimer = import ./lib/store-refresh-timer.nix { };
+  retryInJob = import ./lib/retry-in-job.nix { };
+
+  # How long the controller-token unit waits for Grafana's socket before it
+  # fails into `storeRetry`'s restart. `container@` is ready when the guest's
+  # boot finishes, which can be before Grafana listens.
+  socketWaitSec = 60;
 in
 {
   # What the service IS to every hive is `swarm.grafana` in
@@ -1127,9 +1133,9 @@ in
         SyslogIdentifier = "swarm-bao-grafana-controller-token";
         # Grafana's answers, the new token among them, pass through a temp file.
         PrivateTmp = true;
-        # Each call below is bounded by curl's own timeouts; a run makes up to
-        # about ten of them.
-        TimeoutStartSec = "3min";
+        # The socket wait (`socketWaitSec`) plus the calls below, each bounded
+        # by curl's own timeouts; a run makes up to about ten of them.
+        TimeoutStartSec = "4min";
       };
       environment = {
         BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
@@ -1173,6 +1179,18 @@ in
           exit 1
         }
 
+        ${retryInJob}
+
+        # Any HTTP answer counts; a refused connection is curl's exit status 7.
+        grafana_listening() {
+          curl --silent --output /dev/null --connect-timeout 5 --max-time 15 \
+            --unix-socket ${lib.escapeShellArg socketPath} http://localhost/api/health
+        }
+        if ! retry_in_job ${toString socketWaitSec} grafana_listening; then
+          echo "Grafana's socket ${socketPath} did not answer within ${toString socketWaitSec} seconds." >&2
+          exit 1
+        fi
+
         if [ ! -s ${lib.escapeShellArg adminPasswordPath} ]; then
           echo "no Grafana admin password at ${adminPasswordPath} yet; swarm-bao-grafana-admin writes it." >&2
           exit 1