mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

sum the boot-wait budgets of units that start one after another in a container, and check it

Browse source
This commit is contained in:
atlas 2026-11-02 15:34:52 +01:00 • committed by mara
commit a0c8892b99
3 changed files with 423 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

4
nix/container-modules/swarm-container.nix
View file

@ -15,7 +15,9 @@
# `systemctl start --no-block` (./swarm-journal-forwarder.nix is the pattern).
# `module-eval-swarm-services-switch` fails on a unit whose budget, stated or the
# manager's 90 s default, reaches the limit, unless the unit is listed there with
# a reason.
# a reason. It also adds the budgets of units ordered one after another by
# `after`/`before`, and fails when the longest such chain reaches the limit (a
# unit listed there adds 0 s) or the ordering has a cycle.
{ config, lib, ... }:
let
cfg = config.services.hyperhive.swarmContainer;

178
nix/module-eval/boot-wait.nix
View file

@ -124,43 +124,48 @@ let
|| (unit.preStart or "") != ""
|| (unit.postStart or "") != "";
# What a service's start job can last, and where the number comes from:
# `stated` is the unit's own `TimeoutStartSec`; `default` is the manager's
# `DefaultTimeoutStartSec` (90 s unless the guest sets one), which a unit that
# states none gets; `unbounded` is a oneshot that states none, which systemd
# leaves without a timeout. `seconds = null` is infinity.
budgetOf =
guest: unit:
let
sc = unit.serviceConfig or { };
managerDefault = (guest.systemd.settings.Manager or { }).DefaultTimeoutStartSec or 90;
in
if sc ? TimeoutStartSec then
{
source = "stated";
seconds = spanSeconds sc.TimeoutStartSec;
}
else if (sc.Type or "simple") == "oneshot" then
{
source = "unbounded";
seconds = null;
}
else
{
source = "default";
seconds = spanSeconds managerDefault;
};
# The services in `guest`'s boot wait whose start job can last `limit` seconds
# or more: `{ key; seconds; source; }`, `seconds = null` for infinity. A job
# that ends at `limit` or later can outlast the container's start, because the
# guest has already spent time booting.
#
# `source` says where the number comes from: `stated` is the unit's own
# `TimeoutStartSec`; `default` is the manager's `DefaultTimeoutStartSec` (90 s
# unless the guest sets one), which a unit that states none gets; `unbounded`
# is a oneshot that states none, which systemd leaves without a timeout.
overBudget =
{ guest, limit }:
let
services = guest.systemd.services or { };
managerDefault = (guest.systemd.settings.Manager or { }).DefaultTimeoutStartSec or 90;
in
lib.concatMap (
key:
let
n = lib.removeSuffix ".service" key;
unit = services.${n};
sc = unit.serviceConfig or { };
budget =
if sc ? TimeoutStartSec then
{
source = "stated";
seconds = spanSeconds sc.TimeoutStartSec;
}
else if (sc.Type or "simple") == "oneshot" then
{
source = "unbounded";
seconds = null;
}
else
{
source = "default";
seconds = spanSeconds managerDefault;
};
budget = budgetOf guest unit;
in
lib.optional
(
@ -174,12 +179,139 @@ let
inherit (budget) seconds source;
}
) (bootSet guest.systemd);
# Seconds, `null` being infinity.
addSeconds = a: b: if a == null || b == null then null else a + b;
longer = a: b: b != null && (a == null || a > b);
# The longest chain of the guest's boot-wait units that systemd runs one
# after another, because one names the other in `after` or `before`: a path
# through that ordering where each job-holding service adds its budget (see
# `budgetOf`) and every other unit adds nothing. Only an ordering between two
# units of the boot wait is an edge. `weightless` takes `{ key; seconds;
# source; }` of a job-holding service and says whether it adds 0 s.
#
# Returns `{ cycle; seconds; units; }`. With no cycle in the ordering,
# `cycle = [ ]`, `seconds` is the path's sum and `units` its weighted
# services in order, as `{ key; seconds; source; }`. With a cycle, `cycle` is
# the unit names around it and the other two are `null`: no start order
# exists to add up.
longestChain =
{
guest,
weightless ? (_: false),
}:
let
byName = lib.listToAttrs (map (u: lib.nameValuePair u.name u.unit) (unitsOf guest.systemd));
nodes = lib.filter (key: byName ? ${key}) (bootSet guest.systemd);
isNode = lib.genAttrs nodes (_: true);
ordered = key: field: lib.filter (k: isNode ? ${k}) (byName.${key}.${field} or [ ]);
# `preds.<k>`: the units whose start job ends before `k`'s begins.
preds = lib.genAttrs nodes (
key: lib.unique (ordered key "after" ++ lib.filter (k: lib.elem key (ordered k "before")) nodes)
);
succs = lib.genAttrs nodes (key: lib.filter (k: lib.elem key preds.${k}) nodes);
weightOf =
key:
let
unit = byName.${key};
entry = {
inherit key;
inherit (budgetOf guest unit) seconds source;
};
in
if lib.hasSuffix ".service" key && holdsJob unit && !(weightless entry) then entry else null;
# Taking away, again and again, the units with no predecessor or no
# successor left leaves the units on a cycle and those between cycles.
peeled =
left:
lib.filter (
k: lib.any (p: lib.elem p left) preds.${k} && lib.any (s: lib.elem s left) succs.${k}
) left;
onCycles =
(lib.last (
builtins.genericClosure {
startSet = [
{
key = 0;
left = nodes;
}
];
operator =
s:
let
next = peeled s.left;
in
lib.optional (lib.length next < lib.length s.left) {
key = s.key + 1;
left = next;
};
}
)).left;
# Every unit left has a successor left, so following successors from any
# of them reaches a cycle within that many steps and stays on it.
follow = k: lib.head (lib.filter (s: lib.elem s onCycles) succs.${k});
cycle = map (x: x.key) (
builtins.genericClosure {
startSet = [ { key = lib.foldl' (k: _: follow k) (lib.head onCycles) onCycles; } ];
operator = x: [ { key = follow x.key; } ];
}
);
none = {
seconds = 0;
units = [ ];
};
heavier = acc: c: if longer c.seconds acc.seconds then c else acc;
# `best.<k>`: the heaviest chain ending in `k`.
best = lib.genAttrs nodes (
key:
let
before = lib.foldl' heavier none (map (p: best.${p}) preds.${key});
w = weightOf key;
in
if w == null then
before
else
{
seconds = addSeconds before.seconds w.seconds;
units = before.units ++ [ w ];
}
);
in
if onCycles != [ ] then
{
inherit cycle;
seconds = null;
units = null;
}
else
{
cycle = [ ];
inherit (lib.foldl' heavier none (map (k: best.${k}) nodes)) seconds units;
};
# `a 30s + b 30s = 60s`, an infinite span as `unbounded`.
describeChain =
chain:
let
show = s: if s == null then "unbounded" else "${toString s}s";
in
"${
lib.concatMapStringsSep " + " (u: "${u.key} ${show u.seconds}") chain.units
} = ${show chain.seconds}";
in
{
inherit
spanSeconds
bootSet
holdsJob
budgetOf
overBudget
longestChain
describeChain
;
}

269
nix/module-eval/swarm-services-switch.nix
View file

@ -223,7 +223,8 @@ let
# Boot-wait units that state no `TimeoutStartSec` and are let through, each
# group with the reason. A unit that states a budget is judged on it whatever
# this says. A new unit that states none fails the check until it states one
# or is added here with its reason.
# or is added here with its reason. A unit listed here also adds 0 s to the
# chain of units ordered one after another that `bootWaitChainOffenders` sums.
unstatedAllowed = [
{
why = "NixOS's own units: they change local files, interfaces and rules, and wait on no peer";
@ -283,6 +284,34 @@ let
) (lib.filter (o: o.source == "stated" || !allowedUnstated o.key) (reachingLimit name { }))
) hyperhiveContainers;
# Container `name`'s longest chain of boot-wait units ordered one after
# another, the `unstatedAllowed` ones adding 0 s.
chainOf =
name:
bootWait.longestChain {
guest = serviceContainers.${name}.config;
weightless = o: o.source != "stated" && allowedUnstated o.key;
};
# `<container>: <chain>` where the chain reaches the container's start limit
# with more than one unit in it (a single unit is `bootWaitOffenders`'), or
# `<container>: ordering cycle <unit> -> … -> <unit>`.
bootWaitChainOffenders = lib.concatMap (
name:
let
chain = chainOf name;
limit = startLimitOf name;
in
if chain.cycle != [ ] then
[
"${name}: ordering cycle ${lib.concatStringsSep " -> " (chain.cycle ++ [ (lib.head chain.cycle) ])}"
]
else if lib.length chain.units > 1 && (chain.seconds == null || chain.seconds >= limit) then
[ "${name}: ${bootWait.describeChain chain} >= ${toString limit}s" ]
else
[ ]
) hyperhiveContainers;
# The metrics store on a build older than its socket needs. Only its
# `assertions` are read, and the package is a stub carrying a version.
oldMetricsStore = hive {
@ -1400,14 +1429,246 @@ let
# unit in that queue whose start job can last that long takes the whole
# container down with it. A unit that states no budget has the manager's
# 90 s default, or none if it is a oneshot, and counts too unless
# `unstatedAllowed` names it. Judged per unit, so a chain of units each
# under the limit is not caught. The offenders are in the name, which is
# printed on failure only.
# `unstatedAllowed` names it. Judged per unit; the next case adds the units
# ordered one after another. The offenders are in the name, which is printed
# on failure only.
name = "no unit in a service container's boot wait has a start budget that reaches the container's start limit${
lib.optionalString (bootWaitOffenders != [ ]) ": ${lib.concatStringsSep "; " bootWaitOffenders}"
}";
ok = bootWaitOffenders == [ ];
}
{
# The same limit, judged on the longest chain of units ordered one after
# another (`after`/`before`), the budgets adding up. A chain of one unit is
# the case above; the offenders here have two or more.
name = "no chain of units in a service container's boot wait that start one after another adds up to the container's start limit${
lib.optionalString (
bootWaitChainOffenders != [ ]
) ": ${lib.concatStringsSep "; " bootWaitChainOffenders}"
}";
ok = bootWaitChainOffenders == [ ];
}
{
# The presence arms for the check above: it finds the chains the real
# containers have. hive-matrix's render runs before its sync (20 s + 20 s)
# and authelia's secret fetch before the bridge (30 s + 10 s).
name = "the boot-wait chain check finds the ordered units of the real containers and sums them";
ok =
let
units = name: map (u: u.key) (chainOf name).units;
in
units "hive-matrix" == [
"swarm-matrix-appservice-render.service"
"swarm-matrix-appservice-sync.service"
]
&& (chainOf "hive-matrix").seconds == 40
&&
units "swarm-authelia" == [
"authelia-swarm-secrets.service"
"swarm-authelia-bridge.service"
]
&& (chainOf "swarm-authelia").seconds == 40
&& (chainOf "hive-forge").cycle == [ ]
&& lib.all (n: (chainOf n).cycle == [ ]) hyperhiveContainers;
}
{
# The chain walk on hand-made guests. Budgets add along an ordering,
# written with `after` or with `before`; two units with no ordering run
# side by side and do not add; a unit that adds nothing (a target, a
# service that ends at the fork) passes the ordering on; an ordering to a
# unit outside the boot wait is not a step; the heaviest path of a diamond
# wins; one unbounded unit makes the chain unbounded; and a unit the
# caller calls weightless adds 0.
name = "the boot-wait chain walk adds budgets along after and before, and not across units that run side by side";
ok =
let
svc =
budget: extra:
{
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
TimeoutStartSec = budget;
};
}
// extra;
guestOf = services: {
systemd = {
inherit services;
targets.multi-user.wants = [ "gate.target" ];
targets.gate = { };
};
};
chain =
services: weightless:
bootWait.longestChain {
guest = guestOf services;
inherit weightless;
};
sum = services: (chain services (_: false)).seconds;
keys = services: map (u: u.key) (chain services (_: false)).units;
in
# Two units in sequence: over the limit of 60 s, and under it.
sum {
a = svc 40 { };
b = svc 40 { after = [ "a.service" ]; };
} == 80
&&
sum {
a = svc 20 { };
b = svc 30 { after = [ "a.service" ]; };
} == 50
&&
keys {
a = svc 40 { };
b = svc 40 { after = [ "a.service" ]; };
} == [
"a.service"
"b.service"
]
# `before` is the same ordering from the other side.
&&
sum {
a = svc 40 { before = [ "b.service" ]; };
b = svc 40 { };
} == 80
# Side by side: the larger, not the sum.
&&
sum {
a = svc 40 { };
b = svc 40 { };
} == 40
# Through a unit that adds nothing.
&&
sum {
a = svc 40 { before = [ "gate.target" ]; };
b = svc 40 { after = [ "gate.target" ]; };
} == 80
&&
sum {
a = svc 40 { before = [ "plain.service" ]; };
plain = {
wantedBy = [ "multi-user.target" ];
before = [ "b.service" ];
};
b = svc 40 { };
} == 80
# Outside the boot wait: not a step.
&&
sum {
a = svc 40 { };
b = svc 40 { after = [ "a.service" ]; };
offBoot = {
serviceConfig = {
Type = "oneshot";
TimeoutStartSec = 900;
};
after = [ "b.service" ];
};
} == 80
&&
sum {
a = svc 40 { after = [ "offBoot.service" ]; };
offBoot = {
serviceConfig = {
Type = "oneshot";
TimeoutStartSec = 900;
};
};
} == 40
# A diamond: the heavier side.
&&
keys {
a = svc 10 { };
b = svc 50 { after = [ "a.service" ]; };
c = svc 20 { after = [ "a.service" ]; };
d = svc 5 {
after = [
"b.service"
"c.service"
];
};
} == [
"a.service"
"b.service"
"d.service"
]
# The default (90 s) and an unbounded oneshot add like any other.
&&
sum {
a = {
wantedBy = [ "multi-user.target" ];
preStart = "true";
};
b = svc 10 { after = [ "a.service" ]; };
} == 100
&&
sum {
a = svc 40 { };
b = {
wantedBy = [ "multi-user.target" ];
serviceConfig.Type = "oneshot";
after = [ "a.service" ];
};
} == null
# A weightless unit adds 0 and the rest still add up.
&&
(chain {
a = svc 40 { };
b = svc 40 { after = [ "a.service" ]; };
c = svc 15 { after = [ "b.service" ]; };
} (o: o.key == "b.service")).seconds == 55
&& sum { } == 0;
}
{
# A cycle in the ordering is named, with the units that merely follow it
# left out. The control is the same units without the edge that closes it.
name = "the boot-wait chain walk names a cycle in the ordering";
ok =
let
svc =
extra:
{
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
TimeoutStartSec = 10;
};
}
// extra;
walk =
services:
bootWait.longestChain {
guest.systemd = { inherit services; };
};
cyclic = walk {
a = svc { after = [ "c.service" ]; };
b = svc { after = [ "a.service" ]; };
c = svc { after = [ "b.service" ]; };
tail = svc { after = [ "c.service" ]; };
head = svc { before = [ "a.service" ]; };
};
open = walk {
a = svc { };
b = svc { after = [ "a.service" ]; };
c = svc { after = [ "b.service" ]; };
tail = svc { after = [ "c.service" ]; };
head = svc { before = [ "a.service" ]; };
};
in
lib.sort lib.lessThan cyclic.cycle == [
"a.service"
"b.service"
"c.service"
]
&& cyclic.seconds == null
&& open.cycle == [ ]
&& open.seconds == 50
&&
(walk {
a = svc { after = [ "a.service" ]; };
}).cycle == [ "a.service" ];
}
{
# A name that no container's boot wait carries any more is a dead entry,
# and would let a unit of that name back in unseen. The presence arm is