keep service containers' boot-wait budgets under their start limit, and check it
This commit is contained in:
atlas
mara
parent
da0cdda6a4
commit
70b112058a
1 changed files with 553 additions and 9 deletions
|
|
@ -183,6 +183,110 @@ let
|
|||
};
|
||||
serviceContainers = serviceContainersWithCi.containers;
|
||||
|
||||
bootWait = import ./boot-wait.nix { inherit lib; };
|
||||
|
||||
# Every container hyperhive declares, by name: a generic walk over
|
||||
# `containers` would also judge an operator's own.
|
||||
hyperhiveContainers = [
|
||||
"hive-forge"
|
||||
"hive-matrix"
|
||||
"swarm-attic"
|
||||
"swarm-authelia"
|
||||
"swarm-bao"
|
||||
"swarm-builder"
|
||||
"swarm-ci-runner"
|
||||
"swarm-claude-login"
|
||||
"swarm-grafana"
|
||||
"swarm-nats"
|
||||
"swarm-notify-router"
|
||||
"swarm-otel"
|
||||
"swarm-review-classifier"
|
||||
"swarm-victorialogs"
|
||||
"swarm-victoriametrics"
|
||||
];
|
||||
|
||||
# A container's start limit, in seconds.
|
||||
startLimitOf = name: bootWait.spanSeconds serviceContainers.${name}.timeoutStartSec;
|
||||
|
||||
# The boot-wait units of container `name` whose start job can last `limit`
|
||||
# (default: the container's own start limit) or more.
|
||||
reachingLimit =
|
||||
name:
|
||||
{
|
||||
limit ? startLimitOf name,
|
||||
}:
|
||||
bootWait.overBudget {
|
||||
guest = serviceContainers.${name}.config;
|
||||
inherit limit;
|
||||
};
|
||||
|
||||
# Boot-wait units that state no `TimeoutStartSec` and are let through, each
|
||||
# group with the reason. A unit that states a budget is judged on it whatever
|
||||
# this says. A new unit that states none fails the check until it states one
|
||||
# or is added here with its reason.
|
||||
unstatedAllowed = [
|
||||
{
|
||||
why = "NixOS's own units: they change local files, interfaces and rules, and wait on no peer";
|
||||
units = [
|
||||
"dhcpcd.service"
|
||||
"firewall.service"
|
||||
"linger-users.service"
|
||||
"logrotate-checkconf.service"
|
||||
"network-addresses-eth0.service"
|
||||
"network-local-commands.service"
|
||||
"nscd.service"
|
||||
"resolvconf.service"
|
||||
"suid-sgid-wrappers.service"
|
||||
];
|
||||
}
|
||||
{
|
||||
why = "hyperhive's local oneshots: they write the resolver and trust files, set up local state or initialise the store's own token, and wait on no peer";
|
||||
units = [
|
||||
"forgejo-gpg-init.service"
|
||||
"forgejo-secrets.service"
|
||||
"swarm-bao-token.service"
|
||||
"swarm-bridge-dns.service"
|
||||
"victoriametrics-state-migrate.service"
|
||||
];
|
||||
suffixes = [
|
||||
"-ca-bundle.service"
|
||||
"-ca-bundle-refresh.service"
|
||||
];
|
||||
}
|
||||
{
|
||||
why = "the service's own start: a daemon that does not come up in time is a container that is not useful, whatever its budget";
|
||||
units = [
|
||||
"authelia-swarm.service"
|
||||
"forgejo.service"
|
||||
"gitea-runner-swarm.service"
|
||||
"grafana.service"
|
||||
"nats.service"
|
||||
"opentelemetry-collector.service"
|
||||
"tuwunel.service"
|
||||
"victorialogs.service"
|
||||
"victoriametrics.service"
|
||||
];
|
||||
}
|
||||
{
|
||||
why = "known gap: registering the login source makes an outbound HTTPS call to authelia, and the unit has neither a timeout nor a Restart=";
|
||||
units = [ "forgejo-sso-source.service" ];
|
||||
}
|
||||
];
|
||||
allowedUnstated =
|
||||
key:
|
||||
lib.any (
|
||||
g: lib.elem key (g.units or [ ]) || lib.any (s: lib.hasSuffix s key) (g.suffixes or [ ])
|
||||
) unstatedAllowed;
|
||||
|
||||
# `<container>: <unit> <seconds>s (<source>) >= <limit>s`, over every named
|
||||
# container.
|
||||
bootWaitOffenders = lib.concatMap (
|
||||
name:
|
||||
map (
|
||||
o: "${name}: ${o.key} ${toString o.seconds}s (${o.source}) >= ${toString (startLimitOf name)}s"
|
||||
) (lib.filter (o: o.source == "stated" || !allowedUnstated o.key) (reachingLimit name { }))
|
||||
) hyperhiveContainers;
|
||||
|
||||
# The metrics store on a build older than its socket needs. Only its
|
||||
# `assertions` are read, and the package is a stub carrying a version.
|
||||
oldMetricsStore = hive {
|
||||
|
|
@ -1294,6 +1398,228 @@ let
|
|||
&& lib.all namesNoStoreUnit readersElsewhere
|
||||
&& !(lib.any (u: storeOnlyNoHive.systemd.services ? ${u}) readers);
|
||||
}
|
||||
{
|
||||
# A swarm service container is `Type=notify`: its start job lasts until
|
||||
# the guest's job queue is empty and is killed at `timeoutStartSec`. A
|
||||
# unit in that queue whose start job can last that long takes the whole
|
||||
# container down with it. A unit that states no budget has the manager's
|
||||
# 90 s default, or none if it is a oneshot, and counts too unless
|
||||
# `unstatedAllowed` names it. Judged per unit, so a chain of units each
|
||||
# under the limit is not caught. The offenders are in the name, which is
|
||||
# printed on failure only.
|
||||
name = "no unit in a service container's boot wait has a start budget that reaches the container's start limit${
|
||||
lib.optionalString (bootWaitOffenders != [ ]) ": ${lib.concatStringsSep "; " bootWaitOffenders}"
|
||||
}";
|
||||
ok = bootWaitOffenders == [ ];
|
||||
}
|
||||
{
|
||||
# A name that no container's boot wait carries any more is a dead entry,
|
||||
# and would let a unit of that name back in unseen. The presence arm is
|
||||
# that each entry is found.
|
||||
name = "every unit the boot-wait check lets through without a stated budget is in some container's boot wait";
|
||||
ok =
|
||||
let
|
||||
unstated = lib.concatMap (
|
||||
name: lib.filter (o: o.source != "stated") (reachingLimit name { })
|
||||
) hyperhiveContainers;
|
||||
keys = map (o: o.key) unstated;
|
||||
found = key: lib.elem key keys;
|
||||
foundSuffix = s: lib.any (k: lib.hasSuffix s k) keys;
|
||||
in
|
||||
lib.all (
|
||||
g: lib.all found (g.units or [ ]) && lib.all foundSuffix (g.suffixes or [ ])
|
||||
) unstatedAllowed;
|
||||
}
|
||||
{
|
||||
# Names, not `lib.attrNames containers`: a walk over every container
|
||||
# would also judge an operator's own. Every container the fixture
|
||||
# declares is in the list, so a new one cannot dodge the check.
|
||||
name = "the boot-wait check names every service container the fixture declares";
|
||||
ok =
|
||||
lib.sort lib.lessThan hyperhiveContainers
|
||||
== lib.sort lib.lessThan (lib.attrNames serviceContainers);
|
||||
}
|
||||
{
|
||||
# The presence arms for the check above: it reads real units and real
|
||||
# limits. Each named unit states its budget (30 s, 30 s, 30 s, 20 s, 20 s),
|
||||
# so it is found at a limit equal to it and not one second above (the
|
||||
# boundary), and the container limits are the ones the check reads.
|
||||
name = "the boot-wait check finds the units that state a budget, at the boundary, and reads each container's own limit";
|
||||
ok =
|
||||
let
|
||||
keys = name: limit: map (o: o.key) (reachingLimit name { inherit limit; });
|
||||
mint = "forgejo-swarm-controller-account.service";
|
||||
fetch = "swarm-journal-forwarder-oidc.service";
|
||||
secrets = "authelia-swarm-secrets.service";
|
||||
sync = "swarm-matrix-appservice-sync.service";
|
||||
render = "swarm-matrix-appservice-render.service";
|
||||
in
|
||||
lib.elem mint (keys "hive-forge" 30)
|
||||
&& lib.elem fetch (keys "hive-forge" 30)
|
||||
&& !(lib.elem mint (keys "hive-forge" 31))
|
||||
&& !(lib.elem fetch (keys "hive-forge" 31))
|
||||
&& lib.elem secrets (keys "swarm-authelia" 30)
|
||||
&& !(lib.elem secrets (keys "swarm-authelia" 31))
|
||||
&& lib.elem sync (keys "hive-matrix" 20)
|
||||
&& lib.elem render (keys "hive-matrix" 20)
|
||||
&& !(lib.elem sync (keys "hive-matrix" 21))
|
||||
&& !(lib.elem render (keys "hive-matrix" 21))
|
||||
&& startLimitOf "hive-forge" == 60
|
||||
&& startLimitOf "swarm-bao" == 300;
|
||||
}
|
||||
{
|
||||
# A unit held to a budget that fits the container's start retries with a
|
||||
# growing delay, not at a fixed one, and the authelia bridge's secret
|
||||
# fetch states a budget so that the check above judges it.
|
||||
name = "the store-fetching units in a container's boot wait restart with a growing delay and the bridge's fetch states its budget";
|
||||
ok =
|
||||
let
|
||||
grows =
|
||||
u:
|
||||
let
|
||||
sc = u.serviceConfig;
|
||||
in
|
||||
sc.Restart == "on-failure" && sc.RestartSteps > 0 && sc.RestartMaxDelaySec > sc.RestartSec;
|
||||
authelia = serviceContainers.swarm-authelia.config.systemd.services;
|
||||
in
|
||||
grows serviceContainers.hive-forge.config.systemd.services.forgejo-swarm-controller-account
|
||||
&& grows authelia.authelia-swarm-secrets
|
||||
&& authelia.swarm-authelia-bridge.serviceConfig.TimeoutStartSec == 10;
|
||||
}
|
||||
{
|
||||
# The forwarder's collector is started `--no-block` by the fetch and is
|
||||
# not in the boot wait, whatever its own budget; its fetch is. The
|
||||
# presence arm is the fetch, found by the same walk.
|
||||
name = "the boot-wait walk includes a unit wanted by multi-user.target and leaves out one started on demand";
|
||||
ok =
|
||||
let
|
||||
booted = bootWait.bootSet serviceContainers.hive-forge.config.systemd;
|
||||
in
|
||||
lib.elem "swarm-journal-forwarder-oidc.service" booted
|
||||
&& lib.elem "forgejo-swarm-controller-account.service" booted
|
||||
&& !(lib.elem "opentelemetry-collector.service" booted);
|
||||
}
|
||||
{
|
||||
# The walk and the comparison on a hand-made guest: each unit that must
|
||||
# be found is, and each that must not be is not. Seconds are counted
|
||||
# from every spelling a unit uses, infinity included.
|
||||
name = "the boot-wait check flags a budget at or over the limit in the boot wait, pulled in any way, and nothing else";
|
||||
ok =
|
||||
let
|
||||
oneshot = budget: {
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
TimeoutStartSec = budget;
|
||||
};
|
||||
};
|
||||
guest = {
|
||||
systemd = {
|
||||
targets.multi-user.wants = [ "viaTarget.service" ];
|
||||
services = {
|
||||
wantedBy = oneshot "3min" // {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
};
|
||||
viaRequires = oneshot 120;
|
||||
parent = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
requires = [ "viaRequires.service" ];
|
||||
wants = [ "gate.target" ];
|
||||
};
|
||||
viaTarget = oneshot "1min 30s";
|
||||
viaRequiredBy = oneshot 100 // {
|
||||
requiredBy = [ "multi-user.target" ];
|
||||
};
|
||||
viaGate = oneshot 100 // {
|
||||
wantedBy = [ "gate.target" ];
|
||||
};
|
||||
atLimit = oneshot "1min" // {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
};
|
||||
forever = oneshot "infinity" // {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
};
|
||||
belowLimit = oneshot 59 // {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
};
|
||||
notPulledIn = oneshot 900;
|
||||
orderedOnly = oneshot 900 // {
|
||||
after = [ "wantedBy.service" ];
|
||||
};
|
||||
forks = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
TimeoutStartSec = 900;
|
||||
};
|
||||
};
|
||||
hook = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
preStart = "true";
|
||||
serviceConfig.TimeoutStartSec = "1h";
|
||||
};
|
||||
unstated = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig.Type = "oneshot";
|
||||
};
|
||||
defaulted = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
preStart = "true";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
in
|
||||
lib.sort lib.lessThan (
|
||||
map (o: "${o.key}:${o.source}") (
|
||||
bootWait.overBudget {
|
||||
inherit guest;
|
||||
limit = 60;
|
||||
}
|
||||
)
|
||||
) == [
|
||||
"atLimit.service:stated"
|
||||
"defaulted.service:default"
|
||||
"forever.service:stated"
|
||||
"hook.service:stated"
|
||||
"unstated.service:unbounded"
|
||||
"viaGate.service:stated"
|
||||
"viaRequiredBy.service:stated"
|
||||
"viaRequires.service:stated"
|
||||
"viaTarget.service:stated"
|
||||
"wantedBy.service:stated"
|
||||
]
|
||||
# The default is the manager's: over a limit above it, or with the guest
|
||||
# setting a shorter one, the unit is not reached, and the unbounded
|
||||
# oneshot still is.
|
||||
&& lib.elem "defaulted.service" (
|
||||
map (o: o.key) (
|
||||
bootWait.overBudget {
|
||||
inherit guest;
|
||||
limit = 90;
|
||||
}
|
||||
)
|
||||
)
|
||||
&& !(lib.elem "defaulted.service" (
|
||||
map (o: o.key) (
|
||||
bootWait.overBudget {
|
||||
inherit guest;
|
||||
limit = 91;
|
||||
}
|
||||
)
|
||||
))
|
||||
&& !(lib.elem "defaulted.service" (
|
||||
map (o: o.key) (
|
||||
bootWait.overBudget {
|
||||
guest = guest // {
|
||||
systemd = guest.systemd // {
|
||||
settings.Manager.DefaultTimeoutStartSec = "45s";
|
||||
};
|
||||
};
|
||||
limit = 60;
|
||||
}
|
||||
)
|
||||
));
|
||||
}
|
||||
];
|
||||
in
|
||||
runGroup "swarm-services-switch" cases
|
||||
|
|
|
|||
Loading…
Reference in a new issue