diff --git a/nix/container-modules/swarm-container.nix b/nix/container-modules/swarm-container.nix index fb7a379a..c142b22a 100644 --- a/nix/container-modules/swarm-container.nix +++ b/nix/container-modules/swarm-container.nix @@ -15,7 +15,9 @@ # `systemctl start --no-block` (./swarm-journal-forwarder.nix is the pattern). # `module-eval-swarm-services-switch` fails on a unit whose budget, stated or the # manager's 90 s default, reaches the limit, unless the unit is listed there with -# a reason. +# a reason. It also adds the budgets of units ordered one after another by +# `after`/`before`, and fails when the longest such chain reaches the limit (a +# unit listed there adds 0 s) or the ordering has a cycle. { config, lib, ... }: let cfg = config.services.hyperhive.swarmContainer; diff --git a/nix/module-eval/boot-wait.nix b/nix/module-eval/boot-wait.nix index 35a2c4bf..4ee459a6 100644 --- a/nix/module-eval/boot-wait.nix +++ b/nix/module-eval/boot-wait.nix @@ -124,43 +124,48 @@ let || (unit.preStart or "") != "" || (unit.postStart or "") != ""; + # What a service's start job can last, and where the number comes from: + # `stated` is the unit's own `TimeoutStartSec`; `default` is the manager's + # `DefaultTimeoutStartSec` (90 s unless the guest sets one), which a unit that + # states none gets; `unbounded` is a oneshot that states none, which systemd + # leaves without a timeout. `seconds = null` is infinity. + budgetOf = + guest: unit: + let + sc = unit.serviceConfig or { }; + managerDefault = (guest.systemd.settings.Manager or { }).DefaultTimeoutStartSec or 90; + in + if sc ? TimeoutStartSec then + { + source = "stated"; + seconds = spanSeconds sc.TimeoutStartSec; + } + else if (sc.Type or "simple") == "oneshot" then + { + source = "unbounded"; + seconds = null; + } + else + { + source = "default"; + seconds = spanSeconds managerDefault; + }; + # The services in `guest`'s boot wait whose start job can last `limit` seconds # or more: `{ key; seconds; source; }`, `seconds = null` for infinity. A job # that ends at `limit` or later can outlast the container's start, because the # guest has already spent time booting. - # - # `source` says where the number comes from: `stated` is the unit's own - # `TimeoutStartSec`; `default` is the manager's `DefaultTimeoutStartSec` (90 s - # unless the guest sets one), which a unit that states none gets; `unbounded` - # is a oneshot that states none, which systemd leaves without a timeout. overBudget = { guest, limit }: let services = guest.systemd.services or { }; - managerDefault = (guest.systemd.settings.Manager or { }).DefaultTimeoutStartSec or 90; in lib.concatMap ( key: let n = lib.removeSuffix ".service" key; unit = services.${n}; - sc = unit.serviceConfig or { }; - budget = - if sc ? TimeoutStartSec then - { - source = "stated"; - seconds = spanSeconds sc.TimeoutStartSec; - } - else if (sc.Type or "simple") == "oneshot" then - { - source = "unbounded"; - seconds = null; - } - else - { - source = "default"; - seconds = spanSeconds managerDefault; - }; + budget = budgetOf guest unit; in lib.optional ( @@ -174,12 +179,139 @@ let inherit (budget) seconds source; } ) (bootSet guest.systemd); + + # Seconds, `null` being infinity. + addSeconds = a: b: if a == null || b == null then null else a + b; + longer = a: b: b != null && (a == null || a > b); + + # The longest chain of the guest's boot-wait units that systemd runs one + # after another, because one names the other in `after` or `before`: a path + # through that ordering where each job-holding service adds its budget (see + # `budgetOf`) and every other unit adds nothing. Only an ordering between two + # units of the boot wait is an edge. `weightless` takes `{ key; seconds; + # source; }` of a job-holding service and says whether it adds 0 s. + # + # Returns `{ cycle; seconds; units; }`. With no cycle in the ordering, + # `cycle = [ ]`, `seconds` is the path's sum and `units` its weighted + # services in order, as `{ key; seconds; source; }`. With a cycle, `cycle` is + # the unit names around it and the other two are `null`: no start order + # exists to add up. + longestChain = + { + guest, + weightless ? (_: false), + }: + let + byName = lib.listToAttrs (map (u: lib.nameValuePair u.name u.unit) (unitsOf guest.systemd)); + nodes = lib.filter (key: byName ? ${key}) (bootSet guest.systemd); + isNode = lib.genAttrs nodes (_: true); + ordered = key: field: lib.filter (k: isNode ? ${k}) (byName.${key}.${field} or [ ]); + + # `preds.`: the units whose start job ends before `k`'s begins. + preds = lib.genAttrs nodes ( + key: lib.unique (ordered key "after" ++ lib.filter (k: lib.elem key (ordered k "before")) nodes) + ); + succs = lib.genAttrs nodes (key: lib.filter (k: lib.elem key preds.${k}) nodes); + + weightOf = + key: + let + unit = byName.${key}; + entry = { + inherit key; + inherit (budgetOf guest unit) seconds source; + }; + in + if lib.hasSuffix ".service" key && holdsJob unit && !(weightless entry) then entry else null; + + # Taking away, again and again, the units with no predecessor or no + # successor left leaves the units on a cycle and those between cycles. + peeled = + left: + lib.filter ( + k: lib.any (p: lib.elem p left) preds.${k} && lib.any (s: lib.elem s left) succs.${k} + ) left; + onCycles = + (lib.last ( + builtins.genericClosure { + startSet = [ + { + key = 0; + left = nodes; + } + ]; + operator = + s: + let + next = peeled s.left; + in + lib.optional (lib.length next < lib.length s.left) { + key = s.key + 1; + left = next; + }; + } + )).left; + # Every unit left has a successor left, so following successors from any + # of them reaches a cycle within that many steps and stays on it. + follow = k: lib.head (lib.filter (s: lib.elem s onCycles) succs.${k}); + cycle = map (x: x.key) ( + builtins.genericClosure { + startSet = [ { key = lib.foldl' (k: _: follow k) (lib.head onCycles) onCycles; } ]; + operator = x: [ { key = follow x.key; } ]; + } + ); + + none = { + seconds = 0; + units = [ ]; + }; + heavier = acc: c: if longer c.seconds acc.seconds then c else acc; + # `best.`: the heaviest chain ending in `k`. + best = lib.genAttrs nodes ( + key: + let + before = lib.foldl' heavier none (map (p: best.${p}) preds.${key}); + w = weightOf key; + in + if w == null then + before + else + { + seconds = addSeconds before.seconds w.seconds; + units = before.units ++ [ w ]; + } + ); + in + if onCycles != [ ] then + { + inherit cycle; + seconds = null; + units = null; + } + else + { + cycle = [ ]; + inherit (lib.foldl' heavier none (map (k: best.${k}) nodes)) seconds units; + }; + + # `a 30s + b 30s = 60s`, an infinite span as `unbounded`. + describeChain = + chain: + let + show = s: if s == null then "unbounded" else "${toString s}s"; + in + "${ + lib.concatMapStringsSep " + " (u: "${u.key} ${show u.seconds}") chain.units + } = ${show chain.seconds}"; in { inherit spanSeconds bootSet holdsJob + budgetOf overBudget + longestChain + describeChain ; } diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix index eed1cfbe..7bd0b08b 100644 --- a/nix/module-eval/swarm-services-switch.nix +++ b/nix/module-eval/swarm-services-switch.nix @@ -223,7 +223,8 @@ let # Boot-wait units that state no `TimeoutStartSec` and are let through, each # group with the reason. A unit that states a budget is judged on it whatever # this says. A new unit that states none fails the check until it states one - # or is added here with its reason. + # or is added here with its reason. A unit listed here also adds 0 s to the + # chain of units ordered one after another that `bootWaitChainOffenders` sums. unstatedAllowed = [ { why = "NixOS's own units: they change local files, interfaces and rules, and wait on no peer"; @@ -283,6 +284,34 @@ let ) (lib.filter (o: o.source == "stated" || !allowedUnstated o.key) (reachingLimit name { })) ) hyperhiveContainers; + # Container `name`'s longest chain of boot-wait units ordered one after + # another, the `unstatedAllowed` ones adding 0 s. + chainOf = + name: + bootWait.longestChain { + guest = serviceContainers.${name}.config; + weightless = o: o.source != "stated" && allowedUnstated o.key; + }; + + # `: ` where the chain reaches the container's start limit + # with more than one unit in it (a single unit is `bootWaitOffenders`'), or + # `: ordering cycle -> … -> `. + bootWaitChainOffenders = lib.concatMap ( + name: + let + chain = chainOf name; + limit = startLimitOf name; + in + if chain.cycle != [ ] then + [ + "${name}: ordering cycle ${lib.concatStringsSep " -> " (chain.cycle ++ [ (lib.head chain.cycle) ])}" + ] + else if lib.length chain.units > 1 && (chain.seconds == null || chain.seconds >= limit) then + [ "${name}: ${bootWait.describeChain chain} >= ${toString limit}s" ] + else + [ ] + ) hyperhiveContainers; + # The metrics store on a build older than its socket needs. Only its # `assertions` are read, and the package is a stub carrying a version. oldMetricsStore = hive { @@ -1400,14 +1429,246 @@ let # unit in that queue whose start job can last that long takes the whole # container down with it. A unit that states no budget has the manager's # 90 s default, or none if it is a oneshot, and counts too unless - # `unstatedAllowed` names it. Judged per unit, so a chain of units each - # under the limit is not caught. The offenders are in the name, which is - # printed on failure only. + # `unstatedAllowed` names it. Judged per unit; the next case adds the units + # ordered one after another. The offenders are in the name, which is printed + # on failure only. name = "no unit in a service container's boot wait has a start budget that reaches the container's start limit${ lib.optionalString (bootWaitOffenders != [ ]) ": ${lib.concatStringsSep "; " bootWaitOffenders}" }"; ok = bootWaitOffenders == [ ]; } + { + # The same limit, judged on the longest chain of units ordered one after + # another (`after`/`before`), the budgets adding up. A chain of one unit is + # the case above; the offenders here have two or more. + name = "no chain of units in a service container's boot wait that start one after another adds up to the container's start limit${ + lib.optionalString ( + bootWaitChainOffenders != [ ] + ) ": ${lib.concatStringsSep "; " bootWaitChainOffenders}" + }"; + ok = bootWaitChainOffenders == [ ]; + } + { + # The presence arms for the check above: it finds the chains the real + # containers have. hive-matrix's render runs before its sync (20 s + 20 s) + # and authelia's secret fetch before the bridge (30 s + 10 s). + name = "the boot-wait chain check finds the ordered units of the real containers and sums them"; + ok = + let + units = name: map (u: u.key) (chainOf name).units; + in + units "hive-matrix" == [ + "swarm-matrix-appservice-render.service" + "swarm-matrix-appservice-sync.service" + ] + && (chainOf "hive-matrix").seconds == 40 + && + units "swarm-authelia" == [ + "authelia-swarm-secrets.service" + "swarm-authelia-bridge.service" + ] + && (chainOf "swarm-authelia").seconds == 40 + && (chainOf "hive-forge").cycle == [ ] + && lib.all (n: (chainOf n).cycle == [ ]) hyperhiveContainers; + } + { + # The chain walk on hand-made guests. Budgets add along an ordering, + # written with `after` or with `before`; two units with no ordering run + # side by side and do not add; a unit that adds nothing (a target, a + # service that ends at the fork) passes the ordering on; an ordering to a + # unit outside the boot wait is not a step; the heaviest path of a diamond + # wins; one unbounded unit makes the chain unbounded; and a unit the + # caller calls weightless adds 0. + name = "the boot-wait chain walk adds budgets along after and before, and not across units that run side by side"; + ok = + let + svc = + budget: extra: + { + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "oneshot"; + TimeoutStartSec = budget; + }; + } + // extra; + guestOf = services: { + systemd = { + inherit services; + targets.multi-user.wants = [ "gate.target" ]; + targets.gate = { }; + }; + }; + chain = + services: weightless: + bootWait.longestChain { + guest = guestOf services; + inherit weightless; + }; + sum = services: (chain services (_: false)).seconds; + keys = services: map (u: u.key) (chain services (_: false)).units; + in + # Two units in sequence: over the limit of 60 s, and under it. + sum { + a = svc 40 { }; + b = svc 40 { after = [ "a.service" ]; }; + } == 80 + && + sum { + a = svc 20 { }; + b = svc 30 { after = [ "a.service" ]; }; + } == 50 + && + keys { + a = svc 40 { }; + b = svc 40 { after = [ "a.service" ]; }; + } == [ + "a.service" + "b.service" + ] + # `before` is the same ordering from the other side. + && + sum { + a = svc 40 { before = [ "b.service" ]; }; + b = svc 40 { }; + } == 80 + # Side by side: the larger, not the sum. + && + sum { + a = svc 40 { }; + b = svc 40 { }; + } == 40 + # Through a unit that adds nothing. + && + sum { + a = svc 40 { before = [ "gate.target" ]; }; + b = svc 40 { after = [ "gate.target" ]; }; + } == 80 + && + sum { + a = svc 40 { before = [ "plain.service" ]; }; + plain = { + wantedBy = [ "multi-user.target" ]; + before = [ "b.service" ]; + }; + b = svc 40 { }; + } == 80 + # Outside the boot wait: not a step. + && + sum { + a = svc 40 { }; + b = svc 40 { after = [ "a.service" ]; }; + offBoot = { + serviceConfig = { + Type = "oneshot"; + TimeoutStartSec = 900; + }; + after = [ "b.service" ]; + }; + } == 80 + && + sum { + a = svc 40 { after = [ "offBoot.service" ]; }; + offBoot = { + serviceConfig = { + Type = "oneshot"; + TimeoutStartSec = 900; + }; + }; + } == 40 + # A diamond: the heavier side. + && + keys { + a = svc 10 { }; + b = svc 50 { after = [ "a.service" ]; }; + c = svc 20 { after = [ "a.service" ]; }; + d = svc 5 { + after = [ + "b.service" + "c.service" + ]; + }; + } == [ + "a.service" + "b.service" + "d.service" + ] + # The default (90 s) and an unbounded oneshot add like any other. + && + sum { + a = { + wantedBy = [ "multi-user.target" ]; + preStart = "true"; + }; + b = svc 10 { after = [ "a.service" ]; }; + } == 100 + && + sum { + a = svc 40 { }; + b = { + wantedBy = [ "multi-user.target" ]; + serviceConfig.Type = "oneshot"; + after = [ "a.service" ]; + }; + } == null + # A weightless unit adds 0 and the rest still add up. + && + (chain { + a = svc 40 { }; + b = svc 40 { after = [ "a.service" ]; }; + c = svc 15 { after = [ "b.service" ]; }; + } (o: o.key == "b.service")).seconds == 55 + && sum { } == 0; + } + { + # A cycle in the ordering is named, with the units that merely follow it + # left out. The control is the same units without the edge that closes it. + name = "the boot-wait chain walk names a cycle in the ordering"; + ok = + let + svc = + extra: + { + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "oneshot"; + TimeoutStartSec = 10; + }; + } + // extra; + walk = + services: + bootWait.longestChain { + guest.systemd = { inherit services; }; + }; + cyclic = walk { + a = svc { after = [ "c.service" ]; }; + b = svc { after = [ "a.service" ]; }; + c = svc { after = [ "b.service" ]; }; + tail = svc { after = [ "c.service" ]; }; + head = svc { before = [ "a.service" ]; }; + }; + open = walk { + a = svc { }; + b = svc { after = [ "a.service" ]; }; + c = svc { after = [ "b.service" ]; }; + tail = svc { after = [ "c.service" ]; }; + head = svc { before = [ "a.service" ]; }; + }; + in + lib.sort lib.lessThan cyclic.cycle == [ + "a.service" + "b.service" + "c.service" + ] + && cyclic.seconds == null + && open.cycle == [ ] + && open.seconds == 50 + && + (walk { + a = svc { after = [ "a.service" ]; }; + }).cycle == [ "a.service" ]; + } { # A name that no container's boot wait carries any more is a dead entry, # and would let a unit of that name back in unseen. The presence arm is