keep service containers' boot-wait budgets under their start limit, and check it

nix/module-eval/boot-wait.nix

@@ -0,0 +1,185 @@
+# Which units of a container's guest hold its boot open, and which of those can
+# hold it past the host's start limit for the container.
+#
+# A swarm service container is `Type=notify`: the host's `container@<name>`
+# start job lasts until the guest's job queue is empty, and ends in a kill at
+# `timeoutStartSec`. A guest unit that is pulled in by the boot targets and
+# whose start job blocks therefore counts its start timeout against that limit.
+# Pure functions over a guest's `config.systemd` and the host's
+# `timeoutStartSec` string; ./swarm-services-switch.nix feeds them each named
+# container.
+{ lib }:
+let
+  # A time span as seconds; `null` is infinity, which `TimeoutStartSec` spells
+  # `0`, `"infinity"` or leaves to `Type=oneshot`'s default.
+  spanSeconds =
+    v:
+    if builtins.isInt v then
+      if v == 0 then null else v
+    else if v == "infinity" then
+      null
+    else
+      lib.foldl' (acc: t: acc + tokenSeconds t) 0 (lib.splitString " " (lib.trim v));
+
+  unitSeconds = {
+    s = 1;
+    sec = 1;
+    second = 1;
+    seconds = 1;
+    m = 60;
+    min = 60;
+    minute = 60;
+    minutes = 60;
+    h = 3600;
+    hr = 3600;
+    hour = 3600;
+    hours = 3600;
+  };
+  tokenSeconds =
+    token:
+    let
+      m = builtins.match "([0-9]+)([a-z]*)" token;
+      unit = if m == null then null else lib.elemAt m 1;
+    in
+    if m == null then
+      throw "boot-wait: cannot read '${token}' as a time span"
+    else if unit == "" then
+      lib.toInt (lib.head m)
+    else
+      lib.toInt (lib.head m)
+      * (unitSeconds.${unit} or (throw "boot-wait: unknown time unit '${unit}' in '${token}'"));
+
+  # Where a boot starts: what `multi-user.target` requires, and the targets
+  # the stock units pull in alongside it.
+  rootTargets = [
+    "sysinit.target"
+    "basic.target"
+    "multi-user.target"
+    "sockets.target"
+    "timers.target"
+    "paths.target"
+    "local-fs.target"
+  ];
+
+  kinds = {
+    services = "service";
+    timers = "timer";
+    sockets = "socket";
+    paths = "path";
+    targets = "target";
+  };
+
+  # Every unit the guest configures, as `{ name = "<n>.<type>"; unit; }`.
+  unitsOf =
+    systemd:
+    lib.concatLists (
+      lib.mapAttrsToList (
+        kind: suffix:
+        lib.mapAttrsToList (n: unit: {
+          name = "${n}.${suffix}";
+          inherit unit;
+        }) (systemd.${kind} or { })
+      ) kinds
+    );
+
+  # The units a boot starts: the closure of `rootTargets` over what a unit
+  # `wants`/`requires`/`bindsTo` and over the units naming it in
+  # `wantedBy`/`requiredBy`. `after`/`before` alone pull nothing in.
+  bootSet =
+    systemd:
+    let
+      units = unitsOf systemd;
+      byName = lib.listToAttrs (map (u: lib.nameValuePair u.name u.unit) units);
+      pulls = u: (u.wants or [ ]) ++ (u.requires or [ ]) ++ (u.bindsTo or [ ]);
+      pulledBy = u: (u.wantedBy or [ ]) ++ (u.requiredBy or [ ]);
+      children =
+        key:
+        (pulls (byName.${key} or { }))
+        ++ (map (u: u.name) (lib.filter (u: lib.elem key (pulledBy u.unit)) units));
+    in
+    map (n: n.key) (
+      builtins.genericClosure {
+        startSet = map (key: { inherit key; }) rootTargets;
+        operator = n: map (key: { inherit key; }) (children n.key);
+      }
+    );
+
+  # Whether the service's start job blocks on the unit's own work rather than
+  # ending at the fork: a oneshot, a notifying or forking daemon, or any unit
+  # with a start hook (which runs inside the job).
+  holdsJob =
+    unit:
+    let
+      sc = unit.serviceConfig or { };
+    in
+    lib.elem (sc.Type or "simple") [
+      "oneshot"
+      "notify"
+      "notify-reload"
+      "forking"
+      "dbus"
+    ]
+    || sc ? ExecStartPre
+    || sc ? ExecStartPost
+    || (unit.preStart or "") != ""
+    || (unit.postStart or "") != "";
+
+  # The services in `guest`'s boot wait whose start job can last `limit` seconds
+  # or more: `{ key; seconds; source; }`, `seconds = null` for infinity. A job
+  # that ends at `limit` or later can outlast the container's start, because the
+  # guest has already spent time booting.
+  #
+  # `source` says where the number comes from: `stated` is the unit's own
+  # `TimeoutStartSec`; `default` is the manager's `DefaultTimeoutStartSec` (90 s
+  # unless the guest sets one), which a unit that states none gets; `unbounded`
+  # is a oneshot that states none, which systemd leaves without a timeout.
+  overBudget =
+    { guest, limit }:
+    let
+      services = guest.systemd.services or { };
+      managerDefault = (guest.systemd.settings.Manager or { }).DefaultTimeoutStartSec or 90;
+    in
+    lib.concatMap (
+      key:
+      let
+        n = lib.removeSuffix ".service" key;
+        unit = services.${n};
+        sc = unit.serviceConfig or { };
+        budget =
+          if sc ? TimeoutStartSec then
+            {
+              source = "stated";
+              seconds = spanSeconds sc.TimeoutStartSec;
+            }
+          else if (sc.Type or "simple") == "oneshot" then
+            {
+              source = "unbounded";
+              seconds = null;
+            }
+          else
+            {
+              source = "default";
+              seconds = spanSeconds managerDefault;
+            };
+      in
+      lib.optional
+        (
+          lib.hasSuffix ".service" key
+          && services ? ${n}
+          && holdsJob unit
+          && (budget.seconds == null || budget.seconds >= limit)
+        )
+        {
+          inherit key;
+          inherit (budget) seconds source;
+        }
+    ) (bootSet guest.systemd);
+in
+{
+  inherit
+    spanSeconds
+    bootSet
+    holdsJob
+    overBudget
+    ;
+}