fetch the secrets unit's store leaves concurrently and log how long each phase takes

nix/host-modules/lib/oidc-client-secrets.nix

@@ -27,7 +27,7 @@
   client_secret_read() {
     CLIENT_SECRET_VALUE=""
     [ "$STORE_UP" = 1 ] || return 2
-    if CLIENT_SECRET_VALUE="$(store_bao kv get -field=value "$1" 2>"$STORE_ERR")" \
+    if CLIENT_SECRET_VALUE="$(store_read "$1")" \
       && [ -n "$CLIENT_SECRET_VALUE" ]; then
       return 0
     fi

nix/host-modules/lib/store-seeded-secret.nix

@@ -8,6 +8,11 @@
 # race reads the winner's back. A read of the leaf that fails for any reason
 # but "No value found" prints the store's error and neither mints nor writes.
 #
+# `store_prefetch` reads many leaves concurrently, and the first `store_read` of
+# a leaf returns that answer, status and error included, instead of asking
+# again, so what a caller decides from a read is the same. Later reads of the
+# leaf, such as one that checks a write, ask the store.
+#
 # Pure function — NOT a NixOS module. Call it from a module's `let`:
 #
 #   storeSeededSecret = import ./lib/store-seeded-secret.nix { };
@@ -31,7 +36,8 @@
   store_init() {
     STORE_ERR="$(mktemp)"
     STORE_UP=0
-    trap 'rm -f "$STORE_ERR"' EXIT
+    STORE_PREFETCH="$(mktemp -d)"
+    trap 'rm -rf "$STORE_ERR" "$STORE_PREFETCH"' EXIT
   }
 
   # Sets STORE_UP. A store that cannot be logged in to leaves every secret to
@@ -48,6 +54,41 @@
     fi
   }
 
+  # <leaf>… Starts the first read of each leaf now, at most STORE_PREFETCH_JOBS
+  # at a time, and returns when all have ended. Each read is bounded by
+  # `store_bao`, and its answer is kept for `store_read`. Does nothing without
+  # a login.
+  store_prefetch() {
+    local leaf key
+    [ "$STORE_UP" = 1 ] || return 0
+    for leaf in "$@"; do
+      while [ "$(jobs -rp | wc -l)" -ge "''${STORE_PREFETCH_JOBS:-8}" ]; do
+        wait -n || true
+      done
+      key="$STORE_PREFETCH/$(printf '%s' "$leaf" | tr / _)"
+      (
+        rc=0
+        store_bao kv get -field=value "$leaf" >"$key.out" 2>"$key.err" || rc=$?
+        echo "$rc" >"$key.rc"
+      ) &
+    done
+    wait
+  }
+
+  # <leaf>. `store_bao kv get -field=value <leaf>`, with the error in
+  # $STORE_ERR: the answer `store_prefetch` kept, once, then the store's.
+  store_read() {
+    local key="$STORE_PREFETCH/$(printf '%s' "$1" | tr / _)" rc
+    if [ -e "$key.rc" ]; then
+      rc="$(< "$key.rc")"
+      cat "$key.out"
+      cat "$key.err" >"$STORE_ERR"
+      rm -f "$key.out" "$key.err" "$key.rc"
+      return "$rc"
+    fi
+    store_bao kv get -field=value "$1" 2>"$STORE_ERR"
+  }
+
   # <leaf> <earlier file> <mint command…>. The value on stdout; status 1 when
   # neither the store nor the file can give one. `kv get` exits 2 both for a
   # leaf that holds nothing ("No value found at <path>") and for a refused
@@ -58,7 +99,7 @@
     local leaf="$1" earlier="$2" value candidate rc=0
     shift 2
     if [ "$STORE_UP" = 1 ]; then
-      value="$(store_bao kv get -field=value "$leaf" 2>"$STORE_ERR")" || rc=$?
+      value="$(store_read "$leaf")" || rc=$?
       if [ "$rc" = 0 ] && [ -n "$value" ]; then
         printf '%s' "$value"
         return 0

nix/host-modules/swarm-authelia.nix

@@ -526,12 +526,14 @@ let
   # cannot be read fails the run too, after the files are rendered from the
   # digests (`STORE_UNREADABLE`, checked at the end of the unit's script), so
   # the unit retries until the store answers and authelia has started from the
-  # digests meanwhile.
+  # digests meanwhile. Runs after `storeKeysScript`, which defines
+  # `phase_done` and has already fetched these leaves.
   storeMintScript = ''
     ${oidcClientSecrets}
     ${lib.concatMapStrings (c: ''
       mint_client_secret ${lib.escapeShellArg c.id} ${lib.escapeShellArg (clientLeaf c.id)} || { rc=$?; [ "$rc" = 2 ] || exit "$rc"; STORE_UNREADABLE=1; }
     '') secretClients}
+    phase_done clients
   '';
 
   # The OIDC half of the first-boot generator, kept out of the script
@@ -624,6 +626,12 @@ let
     done
   '';
 
+  # Every leaf the unit reads, for `store_prefetch`: the keys and the client
+  # secrets. The reads that follow are answered from what it fetched.
+  storeLeaves =
+    map keyLeaf (randomKeys ++ lib.optional oidcEnabled "oidc-issuer")
+    ++ lib.optionals oidcEnabled (map (c: clientLeaf c.id) secretClients);
+
   # The same keys from the store, at the same paths. The store's value wins,
   # then the file's, which seeds the store when it holds none, so a key never
   # changes value by moving; a new one is minted only where neither exists.
@@ -632,8 +640,21 @@ let
   storeKeysScript = ''
     ${storeSeededSecret}
     ${refreshConsumer}
+
+    # <name>. One line: how long the phase that ends here took, so a slow store
+    # shows where the time went. Not `SECONDS`, which counts whole seconds.
+    PHASE_AT="$EPOCHREALTIME"
+    phase_done() {
+      local now="$EPOCHREALTIME"
+      echo "phase $1: $(( (''${now//[.,]/} - ''${PHASE_AT//[.,]/}) / 1000 )) ms"
+      PHASE_AT="$now"
+    }
+
     store_init
     store_login
+    phase_done login
+    store_prefetch ${lib.concatMapStringsSep " " lib.escapeShellArg storeLeaves}
+    phase_done reads
 
     mint_hex() {
       head -c 64 /dev/urandom | od -An -tx1 | tr -d ' \n'
@@ -661,6 +682,7 @@ let
       keep_key "$f" mint_hex
     done
     ${lib.optionalString oidcEnabled "keep_key oidc-issuer mint_issuer"}
+    phase_done keys
   '';
 
   # A netns of its own on the hive bridge: the container dials the store, the
@@ -1197,6 +1219,10 @@ in
                 # accepts a connection and never answers: this unit is in the
                 # boot wait of a container with a one-minute start limit.
                 # What a killed run left is picked up by the next one.
+                #
+                # The run's store reads are concurrent (`store_prefetch`): its
+                # length is the login, a few rounds of reads and the writes,
+                # not one round trip per leaf. Raising this raises that wait.
                 TimeoutStartSec = 30;
                 RestartSec = 15;
                 RestartSteps = 3;