sum the boot-wait budgets of units that start one after another in a container, and check it

nix/container-modules/swarm-container.nix

@@ -15,7 +15,9 @@
 # `systemctl start --no-block` (./swarm-journal-forwarder.nix is the pattern).
 # `module-eval-swarm-services-switch` fails on a unit whose budget, stated or the
 # manager's 90 s default, reaches the limit, unless the unit is listed there with
-# a reason.
+# a reason. It also adds the budgets of units ordered one after another by
+# `after`/`before`, and fails when the longest such chain reaches the limit (a
+# unit listed there adds 0 s) or the ordering has a cycle.
 { config, lib, ... }:
 let
   cfg = config.services.hyperhive.swarmContainer;

nix/module-eval/boot-wait.nix

@@ -124,43 +124,48 @@ let
     || (unit.preStart or "") != ""
     || (unit.postStart or "") != "";
 
+  # What a service's start job can last, and where the number comes from:
+  # `stated` is the unit's own `TimeoutStartSec`; `default` is the manager's
+  # `DefaultTimeoutStartSec` (90 s unless the guest sets one), which a unit that
+  # states none gets; `unbounded` is a oneshot that states none, which systemd
+  # leaves without a timeout. `seconds = null` is infinity.
+  budgetOf =
+    guest: unit:
+    let
+      sc = unit.serviceConfig or { };
+      managerDefault = (guest.systemd.settings.Manager or { }).DefaultTimeoutStartSec or 90;
+    in
+    if sc ? TimeoutStartSec then
+      {
+        source = "stated";
+        seconds = spanSeconds sc.TimeoutStartSec;
+      }
+    else if (sc.Type or "simple") == "oneshot" then
+      {
+        source = "unbounded";
+        seconds = null;
+      }
+    else
+      {
+        source = "default";
+        seconds = spanSeconds managerDefault;
+      };
+
   # The services in `guest`'s boot wait whose start job can last `limit` seconds
   # or more: `{ key; seconds; source; }`, `seconds = null` for infinity. A job
   # that ends at `limit` or later can outlast the container's start, because the
   # guest has already spent time booting.
-  #
-  # `source` says where the number comes from: `stated` is the unit's own
-  # `TimeoutStartSec`; `default` is the manager's `DefaultTimeoutStartSec` (90 s
-  # unless the guest sets one), which a unit that states none gets; `unbounded`
-  # is a oneshot that states none, which systemd leaves without a timeout.
   overBudget =
     { guest, limit }:
     let
       services = guest.systemd.services or { };
-      managerDefault = (guest.systemd.settings.Manager or { }).DefaultTimeoutStartSec or 90;
     in
     lib.concatMap (
       key:
       let
         n = lib.removeSuffix ".service" key;
         unit = services.${n};
-        sc = unit.serviceConfig or { };
+        budget = budgetOf guest unit;
-        budget =
-          if sc ? TimeoutStartSec then
-            {
-              source = "stated";
-              seconds = spanSeconds sc.TimeoutStartSec;
-            }
-          else if (sc.Type or "simple") == "oneshot" then
-            {
-              source = "unbounded";
-              seconds = null;
-            }
-          else
-            {
-              source = "default";
-              seconds = spanSeconds managerDefault;
-            };
       in
       lib.optional
         (
@@ -174,12 +179,139 @@ let
           inherit (budget) seconds source;
         }
     ) (bootSet guest.systemd);
+
+  # Seconds, `null` being infinity.
+  addSeconds = a: b: if a == null || b == null then null else a + b;
+  longer = a: b: b != null && (a == null || a > b);
+
+  # The longest chain of the guest's boot-wait units that systemd runs one
+  # after another, because one names the other in `after` or `before`: a path
+  # through that ordering where each job-holding service adds its budget (see
+  # `budgetOf`) and every other unit adds nothing. Only an ordering between two
+  # units of the boot wait is an edge. `weightless` takes `{ key; seconds;
+  # source; }` of a job-holding service and says whether it adds 0 s.
+  #
+  # Returns `{ cycle; seconds; units; }`. With no cycle in the ordering,
+  # `cycle = [ ]`, `seconds` is the path's sum and `units` its weighted
+  # services in order, as `{ key; seconds; source; }`. With a cycle, `cycle` is
+  # the unit names around it and the other two are `null`: no start order
+  # exists to add up.
+  longestChain =
+    {
+      guest,
+      weightless ? (_: false),
+    }:
+    let
+      byName = lib.listToAttrs (map (u: lib.nameValuePair u.name u.unit) (unitsOf guest.systemd));
+      nodes = lib.filter (key: byName ? ${key}) (bootSet guest.systemd);
+      isNode = lib.genAttrs nodes (_: true);
+      ordered = key: field: lib.filter (k: isNode ? ${k}) (byName.${key}.${field} or [ ]);
+
+      # `preds.<k>`: the units whose start job ends before `k`'s begins.
+      preds = lib.genAttrs nodes (
+        key: lib.unique (ordered key "after" ++ lib.filter (k: lib.elem key (ordered k "before")) nodes)
+      );
+      succs = lib.genAttrs nodes (key: lib.filter (k: lib.elem key preds.${k}) nodes);
+
+      weightOf =
+        key:
+        let
+          unit = byName.${key};
+          entry = {
+            inherit key;
+            inherit (budgetOf guest unit) seconds source;
+          };
+        in
+        if lib.hasSuffix ".service" key && holdsJob unit && !(weightless entry) then entry else null;
+
+      # Taking away, again and again, the units with no predecessor or no
+      # successor left leaves the units on a cycle and those between cycles.
+      peeled =
+        left:
+        lib.filter (
+          k: lib.any (p: lib.elem p left) preds.${k} && lib.any (s: lib.elem s left) succs.${k}
+        ) left;
+      onCycles =
+        (lib.last (
+          builtins.genericClosure {
+            startSet = [
+              {
+                key = 0;
+                left = nodes;
+              }
+            ];
+            operator =
+              s:
+              let
+                next = peeled s.left;
+              in
+              lib.optional (lib.length next < lib.length s.left) {
+                key = s.key + 1;
+                left = next;
+              };
+          }
+        )).left;
+      # Every unit left has a successor left, so following successors from any
+      # of them reaches a cycle within that many steps and stays on it.
+      follow = k: lib.head (lib.filter (s: lib.elem s onCycles) succs.${k});
+      cycle = map (x: x.key) (
+        builtins.genericClosure {
+          startSet = [ { key = lib.foldl' (k: _: follow k) (lib.head onCycles) onCycles; } ];
+          operator = x: [ { key = follow x.key; } ];
+        }
+      );
+
+      none = {
+        seconds = 0;
+        units = [ ];
+      };
+      heavier = acc: c: if longer c.seconds acc.seconds then c else acc;
+      # `best.<k>`: the heaviest chain ending in `k`.
+      best = lib.genAttrs nodes (
+        key:
+        let
+          before = lib.foldl' heavier none (map (p: best.${p}) preds.${key});
+          w = weightOf key;
+        in
+        if w == null then
+          before
+        else
+          {
+            seconds = addSeconds before.seconds w.seconds;
+            units = before.units ++ [ w ];
+          }
+      );
+    in
+    if onCycles != [ ] then
+      {
+        inherit cycle;
+        seconds = null;
+        units = null;
+      }
+    else
+      {
+        cycle = [ ];
+        inherit (lib.foldl' heavier none (map (k: best.${k}) nodes)) seconds units;
+      };
+
+  # `a 30s + b 30s = 60s`, an infinite span as `unbounded`.
+  describeChain =
+    chain:
+    let
+      show = s: if s == null then "unbounded" else "${toString s}s";
+    in
+    "${
+      lib.concatMapStringsSep " + " (u: "${u.key} ${show u.seconds}") chain.units
+    } = ${show chain.seconds}";
 in
 {
   inherit
     spanSeconds
     bootSet
     holdsJob
+    budgetOf
     overBudget
+    longestChain
+    describeChain
     ;
 }

nix/module-eval/swarm-services-switch.nix

@@ -223,7 +223,8 @@ let
   # Boot-wait units that state no `TimeoutStartSec` and are let through, each
   # group with the reason. A unit that states a budget is judged on it whatever
   # this says. A new unit that states none fails the check until it states one
-  # or is added here with its reason.
+  # or is added here with its reason. A unit listed here also adds 0 s to the
+  # chain of units ordered one after another that `bootWaitChainOffenders` sums.
   unstatedAllowed = [
     {
       why = "NixOS's own units: they change local files, interfaces and rules, and wait on no peer";
@@ -283,6 +284,34 @@ let
     ) (lib.filter (o: o.source == "stated" || !allowedUnstated o.key) (reachingLimit name { }))
   ) hyperhiveContainers;
 
+  # Container `name`'s longest chain of boot-wait units ordered one after
+  # another, the `unstatedAllowed` ones adding 0 s.
+  chainOf =
+    name:
+    bootWait.longestChain {
+      guest = serviceContainers.${name}.config;
+      weightless = o: o.source != "stated" && allowedUnstated o.key;
+    };
+
+  # `<container>: <chain>` where the chain reaches the container's start limit
+  # with more than one unit in it (a single unit is `bootWaitOffenders`'), or
+  # `<container>: ordering cycle <unit> -> … -> <unit>`.
+  bootWaitChainOffenders = lib.concatMap (
+    name:
+    let
+      chain = chainOf name;
+      limit = startLimitOf name;
+    in
+    if chain.cycle != [ ] then
+      [
+        "${name}: ordering cycle ${lib.concatStringsSep " -> " (chain.cycle ++ [ (lib.head chain.cycle) ])}"
+      ]
+    else if lib.length chain.units > 1 && (chain.seconds == null || chain.seconds >= limit) then
+      [ "${name}: ${bootWait.describeChain chain} >= ${toString limit}s" ]
+    else
+      [ ]
+  ) hyperhiveContainers;
+
   # The metrics store on a build older than its socket needs. Only its
   # `assertions` are read, and the package is a stub carrying a version.
   oldMetricsStore = hive {
@@ -1400,14 +1429,246 @@ let
       # unit in that queue whose start job can last that long takes the whole
       # container down with it. A unit that states no budget has the manager's
       # 90 s default, or none if it is a oneshot, and counts too unless
-      # `unstatedAllowed` names it. Judged per unit, so a chain of units each
+      # `unstatedAllowed` names it. Judged per unit; the next case adds the units
-      # under the limit is not caught. The offenders are in the name, which is
+      # ordered one after another. The offenders are in the name, which is printed
-      # printed on failure only.
+      # on failure only.
       name = "no unit in a service container's boot wait has a start budget that reaches the container's start limit${
         lib.optionalString (bootWaitOffenders != [ ]) ": ${lib.concatStringsSep "; " bootWaitOffenders}"
       }";
       ok = bootWaitOffenders == [ ];
     }
+    {
+      # The same limit, judged on the longest chain of units ordered one after
+      # another (`after`/`before`), the budgets adding up. A chain of one unit is
+      # the case above; the offenders here have two or more.
+      name = "no chain of units in a service container's boot wait that start one after another adds up to the container's start limit${
+        lib.optionalString (
+          bootWaitChainOffenders != [ ]
+        ) ": ${lib.concatStringsSep "; " bootWaitChainOffenders}"
+      }";
+      ok = bootWaitChainOffenders == [ ];
+    }
+    {
+      # The presence arms for the check above: it finds the chains the real
+      # containers have. hive-matrix's render runs before its sync (20 s + 20 s)
+      # and authelia's secret fetch before the bridge (30 s + 10 s).
+      name = "the boot-wait chain check finds the ordered units of the real containers and sums them";
+      ok =
+        let
+          units = name: map (u: u.key) (chainOf name).units;
+        in
+        units "hive-matrix" == [
+          "swarm-matrix-appservice-render.service"
+          "swarm-matrix-appservice-sync.service"
+        ]
+        && (chainOf "hive-matrix").seconds == 40
+        &&
+          units "swarm-authelia" == [
+            "authelia-swarm-secrets.service"
+            "swarm-authelia-bridge.service"
+          ]
+        && (chainOf "swarm-authelia").seconds == 40
+        && (chainOf "hive-forge").cycle == [ ]
+        && lib.all (n: (chainOf n).cycle == [ ]) hyperhiveContainers;
+    }
+    {
+      # The chain walk on hand-made guests. Budgets add along an ordering,
+      # written with `after` or with `before`; two units with no ordering run
+      # side by side and do not add; a unit that adds nothing (a target, a
+      # service that ends at the fork) passes the ordering on; an ordering to a
+      # unit outside the boot wait is not a step; the heaviest path of a diamond
+      # wins; one unbounded unit makes the chain unbounded; and a unit the
+      # caller calls weightless adds 0.
+      name = "the boot-wait chain walk adds budgets along after and before, and not across units that run side by side";
+      ok =
+        let
+          svc =
+            budget: extra:
+            {
+              wantedBy = [ "multi-user.target" ];
+              serviceConfig = {
+                Type = "oneshot";
+                TimeoutStartSec = budget;
+              };
+            }
+            // extra;
+          guestOf = services: {
+            systemd = {
+              inherit services;
+              targets.multi-user.wants = [ "gate.target" ];
+              targets.gate = { };
+            };
+          };
+          chain =
+            services: weightless:
+            bootWait.longestChain {
+              guest = guestOf services;
+              inherit weightless;
+            };
+          sum = services: (chain services (_: false)).seconds;
+          keys = services: map (u: u.key) (chain services (_: false)).units;
+        in
+        # Two units in sequence: over the limit of 60 s, and under it.
+        sum {
+          a = svc 40 { };
+          b = svc 40 { after = [ "a.service" ]; };
+        } == 80
+        &&
+          sum {
+            a = svc 20 { };
+            b = svc 30 { after = [ "a.service" ]; };
+          } == 50
+        &&
+          keys {
+            a = svc 40 { };
+            b = svc 40 { after = [ "a.service" ]; };
+          } == [
+            "a.service"
+            "b.service"
+          ]
+        # `before` is the same ordering from the other side.
+        &&
+          sum {
+            a = svc 40 { before = [ "b.service" ]; };
+            b = svc 40 { };
+          } == 80
+        # Side by side: the larger, not the sum.
+        &&
+          sum {
+            a = svc 40 { };
+            b = svc 40 { };
+          } == 40
+        # Through a unit that adds nothing.
+        &&
+          sum {
+            a = svc 40 { before = [ "gate.target" ]; };
+            b = svc 40 { after = [ "gate.target" ]; };
+          } == 80
+        &&
+          sum {
+            a = svc 40 { before = [ "plain.service" ]; };
+            plain = {
+              wantedBy = [ "multi-user.target" ];
+              before = [ "b.service" ];
+            };
+            b = svc 40 { };
+          } == 80
+        # Outside the boot wait: not a step.
+        &&
+          sum {
+            a = svc 40 { };
+            b = svc 40 { after = [ "a.service" ]; };
+            offBoot = {
+              serviceConfig = {
+                Type = "oneshot";
+                TimeoutStartSec = 900;
+              };
+              after = [ "b.service" ];
+            };
+          } == 80
+        &&
+          sum {
+            a = svc 40 { after = [ "offBoot.service" ]; };
+            offBoot = {
+              serviceConfig = {
+                Type = "oneshot";
+                TimeoutStartSec = 900;
+              };
+            };
+          } == 40
+        # A diamond: the heavier side.
+        &&
+          keys {
+            a = svc 10 { };
+            b = svc 50 { after = [ "a.service" ]; };
+            c = svc 20 { after = [ "a.service" ]; };
+            d = svc 5 {
+              after = [
+                "b.service"
+                "c.service"
+              ];
+            };
+          } == [
+            "a.service"
+            "b.service"
+            "d.service"
+          ]
+        # The default (90 s) and an unbounded oneshot add like any other.
+        &&
+          sum {
+            a = {
+              wantedBy = [ "multi-user.target" ];
+              preStart = "true";
+            };
+            b = svc 10 { after = [ "a.service" ]; };
+          } == 100
+        &&
+          sum {
+            a = svc 40 { };
+            b = {
+              wantedBy = [ "multi-user.target" ];
+              serviceConfig.Type = "oneshot";
+              after = [ "a.service" ];
+            };
+          } == null
+        # A weightless unit adds 0 and the rest still add up.
+        &&
+          (chain {
+            a = svc 40 { };
+            b = svc 40 { after = [ "a.service" ]; };
+            c = svc 15 { after = [ "b.service" ]; };
+          } (o: o.key == "b.service")).seconds == 55
+        && sum { } == 0;
+    }
+    {
+      # A cycle in the ordering is named, with the units that merely follow it
+      # left out. The control is the same units without the edge that closes it.
+      name = "the boot-wait chain walk names a cycle in the ordering";
+      ok =
+        let
+          svc =
+            extra:
+            {
+              wantedBy = [ "multi-user.target" ];
+              serviceConfig = {
+                Type = "oneshot";
+                TimeoutStartSec = 10;
+              };
+            }
+            // extra;
+          walk =
+            services:
+            bootWait.longestChain {
+              guest.systemd = { inherit services; };
+            };
+          cyclic = walk {
+            a = svc { after = [ "c.service" ]; };
+            b = svc { after = [ "a.service" ]; };
+            c = svc { after = [ "b.service" ]; };
+            tail = svc { after = [ "c.service" ]; };
+            head = svc { before = [ "a.service" ]; };
+          };
+          open = walk {
+            a = svc { };
+            b = svc { after = [ "a.service" ]; };
+            c = svc { after = [ "b.service" ]; };
+            tail = svc { after = [ "c.service" ]; };
+            head = svc { before = [ "a.service" ]; };
+          };
+        in
+        lib.sort lib.lessThan cyclic.cycle == [
+          "a.service"
+          "b.service"
+          "c.service"
+        ]
+        && cyclic.seconds == null
+        && open.cycle == [ ]
+        && open.seconds == 50
+        &&
+          (walk {
+            a = svc { after = [ "a.service" ]; };
+          }).cycle == [ "a.service" ];
+    }
     {
       # A name that no container's boot wait carries any more is a dead entry,
       # and would let a unit of that name back in unseen. The presence arm is