mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

collectors: retry a failed start forever with a growing delay instead of hitting start-limit

Browse source
This commit is contained in:
atlas 2026-11-11 00:42:27 +01:00 • committed by mara
commit d48cc83dcb
1 changed files with 101 additions and 74 deletions
Download patch file Download diff file Expand all files Collapse all files

33
nix/module-eval/swarm-otel-core.nix
View file

@ -267,25 +267,28 @@ let
ok = carriesJournaldSeverity (otelSettings otelNoStores).receivers.journald;
}
{
# Sibling of ./hive-otel.nix's identical case for the host-tier
# collector: this one's `oidc/*` extensions call out to authelia at
# startup, so it fails the same way when that restart races its own.
# nixpkgs ships `Restart = "always"` with no `RestartSec`, so without
# this the unit burns its five default attempts inside two seconds and
# lands in `start-limit-hit`, where it stops retrying. The third clause
# is the one that has to hold — `StartLimit*` are `[Unit]` settings
# that systemd ignores under `[Service]`, so a bound written into
# `serviceConfig` renders, deploys and does nothing.
name = "the swarm collector backs off a failed bind from [Unit], not [Service]";
# The `oidc/*` extensions fetch authelia's discovery document through
# the gateway at startup, and the collector exits 1 when that fails. Any
# start limit leaves it failed for good once the gateway has been away
# long enough, so the limit has to be off and the delay has to grow.
# `StartLimit*` are `[Unit]` settings that systemd ignores under
# `[Service]`, so a limit written into `serviceConfig` renders, deploys
# and does nothing. `Restart = "always"` is nixpkgs' own setting: it
# shows the lookup reaches the real unit, so a misspelt attribute path
# cannot pass the other clauses.
name = "the swarm collector retries a failed start forever, with a growing delay";
ok =
let
u = otelNoStores.containers.swarm-otel.config.systemd.services.opentelemetry-collector;
sc = u.serviceConfig;
in
u.serviceConfig.RestartSec or 0 > 0
&& toString u.unitConfig.StartLimitBurst == "12"
&& !(u.serviceConfig ? StartLimitBurst)
# The window has to outlast every attempt, or the burst is unreachable.
&& u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst;
u.serviceConfig.Restart == "always"
&& toString (u.unitConfig.StartLimitIntervalSec or "unset") == "0"
&& !(sc ? StartLimitIntervalSec)
&& !(sc ? StartLimitBurst)
&& sc.RestartSec > 0
&& sc.RestartSteps > 0
&& sc.RestartMaxDelaySec > sc.RestartSec;
}
{
# 🩸 The container's own journal reaches the stores from the collector