mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

sum the boot-wait budgets of units that start one after another in a container, and check it

Browse source
This commit is contained in:
atlas 2026-11-02 15:34:52 +01:00 • committed by mara
commit a0c8892b99
1 changed files with 423 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

269
nix/module-eval/swarm-services-switch.nix
View file

@ -223,7 +223,8 @@ let
# Boot-wait units that state no `TimeoutStartSec` and are let through, each
# group with the reason. A unit that states a budget is judged on it whatever
# this says. A new unit that states none fails the check until it states one
# or is added here with its reason.
# or is added here with its reason. A unit listed here also adds 0 s to the
# chain of units ordered one after another that `bootWaitChainOffenders` sums.
unstatedAllowed = [
{
why = "NixOS's own units: they change local files, interfaces and rules, and wait on no peer";
@ -283,6 +284,34 @@ let
) (lib.filter (o: o.source == "stated" || !allowedUnstated o.key) (reachingLimit name { }))
) hyperhiveContainers;
# Container `name`'s longest chain of boot-wait units ordered one after
# another, the `unstatedAllowed` ones adding 0 s.
chainOf =
name:
bootWait.longestChain {
guest = serviceContainers.${name}.config;
weightless = o: o.source != "stated" && allowedUnstated o.key;
};
# `<container>: <chain>` where the chain reaches the container's start limit
# with more than one unit in it (a single unit is `bootWaitOffenders`'), or
# `<container>: ordering cycle <unit> -> … -> <unit>`.
bootWaitChainOffenders = lib.concatMap (
name:
let
chain = chainOf name;
limit = startLimitOf name;
in
if chain.cycle != [ ] then
[
"${name}: ordering cycle ${lib.concatStringsSep " -> " (chain.cycle ++ [ (lib.head chain.cycle) ])}"
]
else if lib.length chain.units > 1 && (chain.seconds == null || chain.seconds >= limit) then
[ "${name}: ${bootWait.describeChain chain} >= ${toString limit}s" ]
else
[ ]
) hyperhiveContainers;
# The metrics store on a build older than its socket needs. Only its
# `assertions` are read, and the package is a stub carrying a version.
oldMetricsStore = hive {
@ -1400,14 +1429,246 @@ let
# unit in that queue whose start job can last that long takes the whole
# container down with it. A unit that states no budget has the manager's
# 90 s default, or none if it is a oneshot, and counts too unless
# `unstatedAllowed` names it. Judged per unit, so a chain of units each
# under the limit is not caught. The offenders are in the name, which is
# printed on failure only.
# `unstatedAllowed` names it. Judged per unit; the next case adds the units
# ordered one after another. The offenders are in the name, which is printed
# on failure only.
name = "no unit in a service container's boot wait has a start budget that reaches the container's start limit${
lib.optionalString (bootWaitOffenders != [ ]) ": ${lib.concatStringsSep "; " bootWaitOffenders}"
}";
ok = bootWaitOffenders == [ ];
}
{
# The same limit, judged on the longest chain of units ordered one after
# another (`after`/`before`), the budgets adding up. A chain of one unit is
# the case above; the offenders here have two or more.
name = "no chain of units in a service container's boot wait that start one after another adds up to the container's start limit${
lib.optionalString (
bootWaitChainOffenders != [ ]
) ": ${lib.concatStringsSep "; " bootWaitChainOffenders}"
}";
ok = bootWaitChainOffenders == [ ];
}
{
# The presence arms for the check above: it finds the chains the real
# containers have. hive-matrix's render runs before its sync (20 s + 20 s)
# and authelia's secret fetch before the bridge (30 s + 10 s).
name = "the boot-wait chain check finds the ordered units of the real containers and sums them";
ok =
let
units = name: map (u: u.key) (chainOf name).units;
in
units "hive-matrix" == [
"swarm-matrix-appservice-render.service"
"swarm-matrix-appservice-sync.service"
]
&& (chainOf "hive-matrix").seconds == 40
&&
units "swarm-authelia" == [
"authelia-swarm-secrets.service"
"swarm-authelia-bridge.service"
]
&& (chainOf "swarm-authelia").seconds == 40
&& (chainOf "hive-forge").cycle == [ ]
&& lib.all (n: (chainOf n).cycle == [ ]) hyperhiveContainers;
}
{
# The chain walk on hand-made guests. Budgets add along an ordering,
# written with `after` or with `before`; two units with no ordering run
# side by side and do not add; a unit that adds nothing (a target, a
# service that ends at the fork) passes the ordering on; an ordering to a
# unit outside the boot wait is not a step; the heaviest path of a diamond
# wins; one unbounded unit makes the chain unbounded; and a unit the
# caller calls weightless adds 0.
name = "the boot-wait chain walk adds budgets along after and before, and not across units that run side by side";
ok =
let
svc =
budget: extra:
{
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
TimeoutStartSec = budget;
};
}
// extra;
guestOf = services: {
systemd = {
inherit services;
targets.multi-user.wants = [ "gate.target" ];
targets.gate = { };
};
};
chain =
services: weightless:
bootWait.longestChain {
guest = guestOf services;
inherit weightless;
};
sum = services: (chain services (_: false)).seconds;
keys = services: map (u: u.key) (chain services (_: false)).units;
in
# Two units in sequence: over the limit of 60 s, and under it.
sum {
a = svc 40 { };
b = svc 40 { after = [ "a.service" ]; };
} == 80
&&
sum {
a = svc 20 { };
b = svc 30 { after = [ "a.service" ]; };
} == 50
&&
keys {
a = svc 40 { };
b = svc 40 { after = [ "a.service" ]; };
} == [
"a.service"
"b.service"
]
# `before` is the same ordering from the other side.
&&
sum {
a = svc 40 { before = [ "b.service" ]; };
b = svc 40 { };
} == 80
# Side by side: the larger, not the sum.
&&
sum {
a = svc 40 { };
b = svc 40 { };
} == 40
# Through a unit that adds nothing.
&&
sum {
a = svc 40 { before = [ "gate.target" ]; };
b = svc 40 { after = [ "gate.target" ]; };
} == 80
&&
sum {
a = svc 40 { before = [ "plain.service" ]; };
plain = {
wantedBy = [ "multi-user.target" ];
before = [ "b.service" ];
};
b = svc 40 { };
} == 80
# Outside the boot wait: not a step.
&&
sum {
a = svc 40 { };
b = svc 40 { after = [ "a.service" ]; };
offBoot = {
serviceConfig = {
Type = "oneshot";
TimeoutStartSec = 900;
};
after = [ "b.service" ];
};
} == 80
&&
sum {
a = svc 40 { after = [ "offBoot.service" ]; };
offBoot = {
serviceConfig = {
Type = "oneshot";
TimeoutStartSec = 900;
};
};
} == 40
# A diamond: the heavier side.
&&
keys {
a = svc 10 { };
b = svc 50 { after = [ "a.service" ]; };
c = svc 20 { after = [ "a.service" ]; };
d = svc 5 {
after = [
"b.service"
"c.service"
];
};
} == [
"a.service"
"b.service"
"d.service"
]
# The default (90 s) and an unbounded oneshot add like any other.
&&
sum {
a = {
wantedBy = [ "multi-user.target" ];
preStart = "true";
};
b = svc 10 { after = [ "a.service" ]; };
} == 100
&&
sum {
a = svc 40 { };
b = {
wantedBy = [ "multi-user.target" ];
serviceConfig.Type = "oneshot";
after = [ "a.service" ];
};
} == null
# A weightless unit adds 0 and the rest still add up.
&&
(chain {
a = svc 40 { };
b = svc 40 { after = [ "a.service" ]; };
c = svc 15 { after = [ "b.service" ]; };
} (o: o.key == "b.service")).seconds == 55
&& sum { } == 0;
}
{
# A cycle in the ordering is named, with the units that merely follow it
# left out. The control is the same units without the edge that closes it.
name = "the boot-wait chain walk names a cycle in the ordering";
ok =
let
svc =
extra:
{
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
TimeoutStartSec = 10;
};
}
// extra;
walk =
services:
bootWait.longestChain {
guest.systemd = { inherit services; };
};
cyclic = walk {
a = svc { after = [ "c.service" ]; };
b = svc { after = [ "a.service" ]; };
c = svc { after = [ "b.service" ]; };
tail = svc { after = [ "c.service" ]; };
head = svc { before = [ "a.service" ]; };
};
open = walk {
a = svc { };
b = svc { after = [ "a.service" ]; };
c = svc { after = [ "b.service" ]; };
tail = svc { after = [ "c.service" ]; };
head = svc { before = [ "a.service" ]; };
};
in
lib.sort lib.lessThan cyclic.cycle == [
"a.service"
"b.service"
"c.service"
]
&& cyclic.seconds == null
&& open.cycle == [ ]
&& open.seconds == 50
&&
(walk {
a = svc { after = [ "a.service" ]; };
}).cycle == [ "a.service" ];
}
{
# A name that no container's boot wait carries any more is a dead entry,
# and would let a unit of that name back in unseen. The presence arm is