keep service containers' boot-wait budgets under their start limit, and check it
This commit is contained in:
atlas
mara
parent
da0cdda6a4
commit
70b112058a
3 changed files with 553 additions and 9 deletions
185
nix/module-eval/boot-wait.nix
Normal file
185
nix/module-eval/boot-wait.nix
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
# Which units of a container's guest hold its boot open, and which of those can
|
||||
# hold it past the host's start limit for the container.
|
||||
#
|
||||
# A swarm service container is `Type=notify`: the host's `container@<name>`
|
||||
# start job lasts until the guest's job queue is empty, and ends in a kill at
|
||||
# `timeoutStartSec`. A guest unit that is pulled in by the boot targets and
|
||||
# whose start job blocks therefore counts its start timeout against that limit.
|
||||
# Pure functions over a guest's `config.systemd` and the host's
|
||||
# `timeoutStartSec` string; ./swarm-services-switch.nix feeds them each named
|
||||
# container.
|
||||
{ lib }:
|
||||
let
|
||||
# A time span as seconds; `null` is infinity, which `TimeoutStartSec` spells
|
||||
# `0`, `"infinity"` or leaves to `Type=oneshot`'s default.
|
||||
spanSeconds =
|
||||
v:
|
||||
if builtins.isInt v then
|
||||
if v == 0 then null else v
|
||||
else if v == "infinity" then
|
||||
null
|
||||
else
|
||||
lib.foldl' (acc: t: acc + tokenSeconds t) 0 (lib.splitString " " (lib.trim v));
|
||||
|
||||
unitSeconds = {
|
||||
s = 1;
|
||||
sec = 1;
|
||||
second = 1;
|
||||
seconds = 1;
|
||||
m = 60;
|
||||
min = 60;
|
||||
minute = 60;
|
||||
minutes = 60;
|
||||
h = 3600;
|
||||
hr = 3600;
|
||||
hour = 3600;
|
||||
hours = 3600;
|
||||
};
|
||||
tokenSeconds =
|
||||
token:
|
||||
let
|
||||
m = builtins.match "([0-9]+)([a-z]*)" token;
|
||||
unit = if m == null then null else lib.elemAt m 1;
|
||||
in
|
||||
if m == null then
|
||||
throw "boot-wait: cannot read '${token}' as a time span"
|
||||
else if unit == "" then
|
||||
lib.toInt (lib.head m)
|
||||
else
|
||||
lib.toInt (lib.head m)
|
||||
* (unitSeconds.${unit} or (throw "boot-wait: unknown time unit '${unit}' in '${token}'"));
|
||||
|
||||
# Where a boot starts: what `multi-user.target` requires, and the targets
|
||||
# the stock units pull in alongside it.
|
||||
rootTargets = [
|
||||
"sysinit.target"
|
||||
"basic.target"
|
||||
"multi-user.target"
|
||||
"sockets.target"
|
||||
"timers.target"
|
||||
"paths.target"
|
||||
"local-fs.target"
|
||||
];
|
||||
|
||||
kinds = {
|
||||
services = "service";
|
||||
timers = "timer";
|
||||
sockets = "socket";
|
||||
paths = "path";
|
||||
targets = "target";
|
||||
};
|
||||
|
||||
# Every unit the guest configures, as `{ name = "<n>.<type>"; unit; }`.
|
||||
unitsOf =
|
||||
systemd:
|
||||
lib.concatLists (
|
||||
lib.mapAttrsToList (
|
||||
kind: suffix:
|
||||
lib.mapAttrsToList (n: unit: {
|
||||
name = "${n}.${suffix}";
|
||||
inherit unit;
|
||||
}) (systemd.${kind} or { })
|
||||
) kinds
|
||||
);
|
||||
|
||||
# The units a boot starts: the closure of `rootTargets` over what a unit
|
||||
# `wants`/`requires`/`bindsTo` and over the units naming it in
|
||||
# `wantedBy`/`requiredBy`. `after`/`before` alone pull nothing in.
|
||||
bootSet =
|
||||
systemd:
|
||||
let
|
||||
units = unitsOf systemd;
|
||||
byName = lib.listToAttrs (map (u: lib.nameValuePair u.name u.unit) units);
|
||||
pulls = u: (u.wants or [ ]) ++ (u.requires or [ ]) ++ (u.bindsTo or [ ]);
|
||||
pulledBy = u: (u.wantedBy or [ ]) ++ (u.requiredBy or [ ]);
|
||||
children =
|
||||
key:
|
||||
(pulls (byName.${key} or { }))
|
||||
++ (map (u: u.name) (lib.filter (u: lib.elem key (pulledBy u.unit)) units));
|
||||
in
|
||||
map (n: n.key) (
|
||||
builtins.genericClosure {
|
||||
startSet = map (key: { inherit key; }) rootTargets;
|
||||
operator = n: map (key: { inherit key; }) (children n.key);
|
||||
}
|
||||
);
|
||||
|
||||
# Whether the service's start job blocks on the unit's own work rather than
|
||||
# ending at the fork: a oneshot, a notifying or forking daemon, or any unit
|
||||
# with a start hook (which runs inside the job).
|
||||
holdsJob =
|
||||
unit:
|
||||
let
|
||||
sc = unit.serviceConfig or { };
|
||||
in
|
||||
lib.elem (sc.Type or "simple") [
|
||||
"oneshot"
|
||||
"notify"
|
||||
"notify-reload"
|
||||
"forking"
|
||||
"dbus"
|
||||
]
|
||||
|| sc ? ExecStartPre
|
||||
|| sc ? ExecStartPost
|
||||
|| (unit.preStart or "") != ""
|
||||
|| (unit.postStart or "") != "";
|
||||
|
||||
# The services in `guest`'s boot wait whose start job can last `limit` seconds
|
||||
# or more: `{ key; seconds; source; }`, `seconds = null` for infinity. A job
|
||||
# that ends at `limit` or later can outlast the container's start, because the
|
||||
# guest has already spent time booting.
|
||||
#
|
||||
# `source` says where the number comes from: `stated` is the unit's own
|
||||
# `TimeoutStartSec`; `default` is the manager's `DefaultTimeoutStartSec` (90 s
|
||||
# unless the guest sets one), which a unit that states none gets; `unbounded`
|
||||
# is a oneshot that states none, which systemd leaves without a timeout.
|
||||
overBudget =
|
||||
{ guest, limit }:
|
||||
let
|
||||
services = guest.systemd.services or { };
|
||||
managerDefault = (guest.systemd.settings.Manager or { }).DefaultTimeoutStartSec or 90;
|
||||
in
|
||||
lib.concatMap (
|
||||
key:
|
||||
let
|
||||
n = lib.removeSuffix ".service" key;
|
||||
unit = services.${n};
|
||||
sc = unit.serviceConfig or { };
|
||||
budget =
|
||||
if sc ? TimeoutStartSec then
|
||||
{
|
||||
source = "stated";
|
||||
seconds = spanSeconds sc.TimeoutStartSec;
|
||||
}
|
||||
else if (sc.Type or "simple") == "oneshot" then
|
||||
{
|
||||
source = "unbounded";
|
||||
seconds = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
source = "default";
|
||||
seconds = spanSeconds managerDefault;
|
||||
};
|
||||
in
|
||||
lib.optional
|
||||
(
|
||||
lib.hasSuffix ".service" key
|
||||
&& services ? ${n}
|
||||
&& holdsJob unit
|
||||
&& (budget.seconds == null || budget.seconds >= limit)
|
||||
)
|
||||
{
|
||||
inherit key;
|
||||
inherit (budget) seconds source;
|
||||
}
|
||||
) (bootSet guest.systemd);
|
||||
in
|
||||
{
|
||||
inherit
|
||||
spanSeconds
|
||||
bootSet
|
||||
holdsJob
|
||||
overBudget
|
||||
;
|
||||
}
|
||||
|
|
@ -409,6 +409,7 @@ let
|
|||
&& lib.hasInfix "${toString budget} account_listed; then" s
|
||||
&& budget > 0
|
||||
&& budget <= 20
|
||||
&& budget < u.serviceConfig.TimeoutStartSec
|
||||
&& allLocal.containers.hive-forge.timeoutStartSec == "1min"
|
||||
&& lib.elem "multi-user.target" u.wantedBy
|
||||
&& lib.hasInfix "list=\"$(forgejo admin user list \"$@\")\"" s
|
||||
|
|
|
|||
|
|
@ -183,6 +183,110 @@ let
|
|||
};
|
||||
serviceContainers = serviceContainersWithCi.containers;
|
||||
|
||||
bootWait = import ./boot-wait.nix { inherit lib; };
|
||||
|
||||
# Every container hyperhive declares, by name: a generic walk over
|
||||
# `containers` would also judge an operator's own.
|
||||
hyperhiveContainers = [
|
||||
"hive-forge"
|
||||
"hive-matrix"
|
||||
"swarm-attic"
|
||||
"swarm-authelia"
|
||||
"swarm-bao"
|
||||
"swarm-builder"
|
||||
"swarm-ci-runner"
|
||||
"swarm-claude-login"
|
||||
"swarm-grafana"
|
||||
"swarm-nats"
|
||||
"swarm-notify-router"
|
||||
"swarm-otel"
|
||||
"swarm-review-classifier"
|
||||
"swarm-victorialogs"
|
||||
"swarm-victoriametrics"
|
||||
];
|
||||
|
||||
# A container's start limit, in seconds.
|
||||
startLimitOf = name: bootWait.spanSeconds serviceContainers.${name}.timeoutStartSec;
|
||||
|
||||
# The boot-wait units of container `name` whose start job can last `limit`
|
||||
# (default: the container's own start limit) or more.
|
||||
reachingLimit =
|
||||
name:
|
||||
{
|
||||
limit ? startLimitOf name,
|
||||
}:
|
||||
bootWait.overBudget {
|
||||
guest = serviceContainers.${name}.config;
|
||||
inherit limit;
|
||||
};
|
||||
|
||||
# Boot-wait units that state no `TimeoutStartSec` and are let through, each
|
||||
# group with the reason. A unit that states a budget is judged on it whatever
|
||||
# this says. A new unit that states none fails the check until it states one
|
||||
# or is added here with its reason.
|
||||
unstatedAllowed = [
|
||||
{
|
||||
why = "NixOS's own units: they change local files, interfaces and rules, and wait on no peer";
|
||||
units = [
|
||||
"dhcpcd.service"
|
||||
"firewall.service"
|
||||
"linger-users.service"
|
||||
"logrotate-checkconf.service"
|
||||
"network-addresses-eth0.service"
|
||||
"network-local-commands.service"
|
||||
"nscd.service"
|
||||
"resolvconf.service"
|
||||
"suid-sgid-wrappers.service"
|
||||
];
|
||||
}
|
||||
{
|
||||
why = "hyperhive's local oneshots: they write the resolver and trust files, set up local state or initialise the store's own token, and wait on no peer";
|
||||
units = [
|
||||
"forgejo-gpg-init.service"
|
||||
"forgejo-secrets.service"
|
||||
"swarm-bao-token.service"
|
||||
"swarm-bridge-dns.service"
|
||||
"victoriametrics-state-migrate.service"
|
||||
];
|
||||
suffixes = [
|
||||
"-ca-bundle.service"
|
||||
"-ca-bundle-refresh.service"
|
||||
];
|
||||
}
|
||||
{
|
||||
why = "the service's own start: a daemon that does not come up in time is a container that is not useful, whatever its budget";
|
||||
units = [
|
||||
"authelia-swarm.service"
|
||||
"forgejo.service"
|
||||
"gitea-runner-swarm.service"
|
||||
"grafana.service"
|
||||
"nats.service"
|
||||
"opentelemetry-collector.service"
|
||||
"tuwunel.service"
|
||||
"victorialogs.service"
|
||||
"victoriametrics.service"
|
||||
];
|
||||
}
|
||||
{
|
||||
why = "known gap: registering the login source makes an outbound HTTPS call to authelia, and the unit has neither a timeout nor a Restart=";
|
||||
units = [ "forgejo-sso-source.service" ];
|
||||
}
|
||||
];
|
||||
allowedUnstated =
|
||||
key:
|
||||
lib.any (
|
||||
g: lib.elem key (g.units or [ ]) || lib.any (s: lib.hasSuffix s key) (g.suffixes or [ ])
|
||||
) unstatedAllowed;
|
||||
|
||||
# `<container>: <unit> <seconds>s (<source>) >= <limit>s`, over every named
|
||||
# container.
|
||||
bootWaitOffenders = lib.concatMap (
|
||||
name:
|
||||
map (
|
||||
o: "${name}: ${o.key} ${toString o.seconds}s (${o.source}) >= ${toString (startLimitOf name)}s"
|
||||
) (lib.filter (o: o.source == "stated" || !allowedUnstated o.key) (reachingLimit name { }))
|
||||
) hyperhiveContainers;
|
||||
|
||||
# The metrics store on a build older than its socket needs. Only its
|
||||
# `assertions` are read, and the package is a stub carrying a version.
|
||||
oldMetricsStore = hive {
|
||||
|
|
@ -1294,6 +1398,228 @@ let
|
|||
&& lib.all namesNoStoreUnit readersElsewhere
|
||||
&& !(lib.any (u: storeOnlyNoHive.systemd.services ? ${u}) readers);
|
||||
}
|
||||
{
|
||||
# A swarm service container is `Type=notify`: its start job lasts until
|
||||
# the guest's job queue is empty and is killed at `timeoutStartSec`. A
|
||||
# unit in that queue whose start job can last that long takes the whole
|
||||
# container down with it. A unit that states no budget has the manager's
|
||||
# 90 s default, or none if it is a oneshot, and counts too unless
|
||||
# `unstatedAllowed` names it. Judged per unit, so a chain of units each
|
||||
# under the limit is not caught. The offenders are in the name, which is
|
||||
# printed on failure only.
|
||||
name = "no unit in a service container's boot wait has a start budget that reaches the container's start limit${
|
||||
lib.optionalString (bootWaitOffenders != [ ]) ": ${lib.concatStringsSep "; " bootWaitOffenders}"
|
||||
}";
|
||||
ok = bootWaitOffenders == [ ];
|
||||
}
|
||||
{
|
||||
# A name that no container's boot wait carries any more is a dead entry,
|
||||
# and would let a unit of that name back in unseen. The presence arm is
|
||||
# that each entry is found.
|
||||
name = "every unit the boot-wait check lets through without a stated budget is in some container's boot wait";
|
||||
ok =
|
||||
let
|
||||
unstated = lib.concatMap (
|
||||
name: lib.filter (o: o.source != "stated") (reachingLimit name { })
|
||||
) hyperhiveContainers;
|
||||
keys = map (o: o.key) unstated;
|
||||
found = key: lib.elem key keys;
|
||||
foundSuffix = s: lib.any (k: lib.hasSuffix s k) keys;
|
||||
in
|
||||
lib.all (
|
||||
g: lib.all found (g.units or [ ]) && lib.all foundSuffix (g.suffixes or [ ])
|
||||
) unstatedAllowed;
|
||||
}
|
||||
{
|
||||
# Names, not `lib.attrNames containers`: a walk over every container
|
||||
# would also judge an operator's own. Every container the fixture
|
||||
# declares is in the list, so a new one cannot dodge the check.
|
||||
name = "the boot-wait check names every service container the fixture declares";
|
||||
ok =
|
||||
lib.sort lib.lessThan hyperhiveContainers
|
||||
== lib.sort lib.lessThan (lib.attrNames serviceContainers);
|
||||
}
|
||||
{
|
||||
# The presence arms for the check above: it reads real units and real
|
||||
# limits. Each named unit states its budget (30 s, 30 s, 30 s, 20 s, 20 s),
|
||||
# so it is found at a limit equal to it and not one second above (the
|
||||
# boundary), and the container limits are the ones the check reads.
|
||||
name = "the boot-wait check finds the units that state a budget, at the boundary, and reads each container's own limit";
|
||||
ok =
|
||||
let
|
||||
keys = name: limit: map (o: o.key) (reachingLimit name { inherit limit; });
|
||||
mint = "forgejo-swarm-controller-account.service";
|
||||
fetch = "swarm-journal-forwarder-oidc.service";
|
||||
secrets = "authelia-swarm-secrets.service";
|
||||
sync = "swarm-matrix-appservice-sync.service";
|
||||
render = "swarm-matrix-appservice-render.service";
|
||||
in
|
||||
lib.elem mint (keys "hive-forge" 30)
|
||||
&& lib.elem fetch (keys "hive-forge" 30)
|
||||
&& !(lib.elem mint (keys "hive-forge" 31))
|
||||
&& !(lib.elem fetch (keys "hive-forge" 31))
|
||||
&& lib.elem secrets (keys "swarm-authelia" 30)
|
||||
&& !(lib.elem secrets (keys "swarm-authelia" 31))
|
||||
&& lib.elem sync (keys "hive-matrix" 20)
|
||||
&& lib.elem render (keys "hive-matrix" 20)
|
||||
&& !(lib.elem sync (keys "hive-matrix" 21))
|
||||
&& !(lib.elem render (keys "hive-matrix" 21))
|
||||
&& startLimitOf "hive-forge" == 60
|
||||
&& startLimitOf "swarm-bao" == 300;
|
||||
}
|
||||
{
|
||||
# A unit held to a budget that fits the container's start retries with a
|
||||
# growing delay, not at a fixed one, and the authelia bridge's secret
|
||||
# fetch states a budget so that the check above judges it.
|
||||
name = "the store-fetching units in a container's boot wait restart with a growing delay and the bridge's fetch states its budget";
|
||||
ok =
|
||||
let
|
||||
grows =
|
||||
u:
|
||||
let
|
||||
sc = u.serviceConfig;
|
||||
in
|
||||
sc.Restart == "on-failure" && sc.RestartSteps > 0 && sc.RestartMaxDelaySec > sc.RestartSec;
|
||||
authelia = serviceContainers.swarm-authelia.config.systemd.services;
|
||||
in
|
||||
grows serviceContainers.hive-forge.config.systemd.services.forgejo-swarm-controller-account
|
||||
&& grows authelia.authelia-swarm-secrets
|
||||
&& authelia.swarm-authelia-bridge.serviceConfig.TimeoutStartSec == 10;
|
||||
}
|
||||
{
|
||||
# The forwarder's collector is started `--no-block` by the fetch and is
|
||||
# not in the boot wait, whatever its own budget; its fetch is. The
|
||||
# presence arm is the fetch, found by the same walk.
|
||||
name = "the boot-wait walk includes a unit wanted by multi-user.target and leaves out one started on demand";
|
||||
ok =
|
||||
let
|
||||
booted = bootWait.bootSet serviceContainers.hive-forge.config.systemd;
|
||||
in
|
||||
lib.elem "swarm-journal-forwarder-oidc.service" booted
|
||||
&& lib.elem "forgejo-swarm-controller-account.service" booted
|
||||
&& !(lib.elem "opentelemetry-collector.service" booted);
|
||||
}
|
||||
{
|
||||
# The walk and the comparison on a hand-made guest: each unit that must
|
||||
# be found is, and each that must not be is not. Seconds are counted
|
||||
# from every spelling a unit uses, infinity included.
|
||||
name = "the boot-wait check flags a budget at or over the limit in the boot wait, pulled in any way, and nothing else";
|
||||
ok =
|
||||
let
|
||||
oneshot = budget: {
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
TimeoutStartSec = budget;
|
||||
};
|
||||
};
|
||||
guest = {
|
||||
systemd = {
|
||||
targets.multi-user.wants = [ "viaTarget.service" ];
|
||||
services = {
|
||||
wantedBy = oneshot "3min" // {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
};
|
||||
viaRequires = oneshot 120;
|
||||
parent = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
requires = [ "viaRequires.service" ];
|
||||
wants = [ "gate.target" ];
|
||||
};
|
||||
viaTarget = oneshot "1min 30s";
|
||||
viaRequiredBy = oneshot 100 // {
|
||||
requiredBy = [ "multi-user.target" ];
|
||||
};
|
||||
viaGate = oneshot 100 // {
|
||||
wantedBy = [ "gate.target" ];
|
||||
};
|
||||
atLimit = oneshot "1min" // {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
};
|
||||
forever = oneshot "infinity" // {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
};
|
||||
belowLimit = oneshot 59 // {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
};
|
||||
notPulledIn = oneshot 900;
|
||||
orderedOnly = oneshot 900 // {
|
||||
after = [ "wantedBy.service" ];
|
||||
};
|
||||
forks = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
TimeoutStartSec = 900;
|
||||
};
|
||||
};
|
||||
hook = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
preStart = "true";
|
||||
serviceConfig.TimeoutStartSec = "1h";
|
||||
};
|
||||
unstated = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig.Type = "oneshot";
|
||||
};
|
||||
defaulted = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
preStart = "true";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
in
|
||||
lib.sort lib.lessThan (
|
||||
map (o: "${o.key}:${o.source}") (
|
||||
bootWait.overBudget {
|
||||
inherit guest;
|
||||
limit = 60;
|
||||
}
|
||||
)
|
||||
) == [
|
||||
"atLimit.service:stated"
|
||||
"defaulted.service:default"
|
||||
"forever.service:stated"
|
||||
"hook.service:stated"
|
||||
"unstated.service:unbounded"
|
||||
"viaGate.service:stated"
|
||||
"viaRequiredBy.service:stated"
|
||||
"viaRequires.service:stated"
|
||||
"viaTarget.service:stated"
|
||||
"wantedBy.service:stated"
|
||||
]
|
||||
# The default is the manager's: over a limit above it, or with the guest
|
||||
# setting a shorter one, the unit is not reached, and the unbounded
|
||||
# oneshot still is.
|
||||
&& lib.elem "defaulted.service" (
|
||||
map (o: o.key) (
|
||||
bootWait.overBudget {
|
||||
inherit guest;
|
||||
limit = 90;
|
||||
}
|
||||
)
|
||||
)
|
||||
&& !(lib.elem "defaulted.service" (
|
||||
map (o: o.key) (
|
||||
bootWait.overBudget {
|
||||
inherit guest;
|
||||
limit = 91;
|
||||
}
|
||||
)
|
||||
))
|
||||
&& !(lib.elem "defaulted.service" (
|
||||
map (o: o.key) (
|
||||
bootWait.overBudget {
|
||||
guest = guest // {
|
||||
systemd = guest.systemd // {
|
||||
settings.Manager.DefaultTimeoutStartSec = "45s";
|
||||
};
|
||||
};
|
||||
limit = 60;
|
||||
}
|
||||
)
|
||||
));
|
||||
}
|
||||
];
|
||||
in
|
||||
runGroup "swarm-services-switch" cases
|
||||
|
|
|
|||
Loading…
Reference in a new issue