mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

collectors: retry a failed start forever with a growing delay instead of hitting start-limit

Browse source
This commit is contained in:
atlas 2026-11-11 00:42:27 +01:00 • committed by mara
commit d48cc83dcb
2 changed files with 101 additions and 74 deletions
Download patch file Download diff file Expand all files Collapse all files

46
nix/module-eval/hive-otel.nix
View file

@ -190,25 +190,28 @@ let
!(m ? address) && (lib.head m.readers).pull.exporter.prometheus.port == 8888;
}
{
# A bind that loses the race with the outgoing process is the whole
# failure: nixpkgs ships `Restart = "always"` with no `RestartSec`, so
# without this the unit spends its five default attempts inside two
# seconds and lands in `start-limit-hit`, where it stops retrying. The
# third clause is the one that has to hold — `StartLimit*` are `[Unit]`
# settings that systemd ignores under `[Service]`, so a bound written
# into `serviceConfig` renders, deploys and does nothing. Asserted
# where nixpkgs puts it rather than where it was written, same as
# ./bao-grants.nix.
name = "the hive collector backs off a failed bind from [Unit], not [Service]";
# A failed start (a bind that loses the race with the outgoing process,
# a credential not yet delivered) must keep retrying: any start limit
# leaves the collector failed until someone starts it by hand.
# `StartLimit*` are `[Unit]` settings that systemd ignores under
# `[Service]`, so a limit written into `serviceConfig` renders, deploys
# and does nothing. Asserted where nixpkgs puts it rather than where it
# was written, same as ./bao-grants.nix. `Restart = "always"` is
# nixpkgs' own setting: it shows the lookup reaches the real unit, so a
# misspelt attribute path cannot pass the other clauses.
name = "the hive collector retries a failed start forever, with a growing delay";
ok =
let
u = hiveOtel.systemd.services.opentelemetry-collector;
sc = u.serviceConfig;
in
u.serviceConfig.RestartSec or 0 > 0
&& toString u.unitConfig.StartLimitBurst == "12"
&& !(u.serviceConfig ? StartLimitBurst)
# The window has to outlast every attempt, or the burst is unreachable.
&& u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst;
sc.Restart == "always"
&& toString (u.unitConfig.StartLimitIntervalSec or "unset") == "0"
&& !(sc ? StartLimitIntervalSec)
&& !(sc ? StartLimitBurst)
&& sc.RestartSec > 0
&& sc.RestartSteps > 0
&& sc.RestartMaxDelaySec > sc.RestartSec;
}
{
# Collectors share a network namespace whenever they are co-located,
@ -319,14 +322,15 @@ let
&& lib.hasInfix ''if [ "$written_us" -le "$started_us" ]; then'' s;
}
{
# On a first boot the file does not exist until the fetch succeeds, so
# the collector fails `LoadCredential` into its start limit, i.e. the
# `failed` state. Only `reset-failed` clears that limit, so the fetch
# has to reset before it starts; `try-restart` acts on no failed unit.
# On a first boot the file does not exist until the fetch succeeds, and
# the collector retries `LoadCredential` with a growing delay. A
# collector that is `failed` for another reason is not retried by
# systemd, and `try-restart` acts on no failed unit, so the fetch has to
# `reset-failed` before it starts.
# A never-active collector has an empty `ActiveEnterTimestamp`, read as
# 0, so the file always counts as newer. The collector only `wants` the
# fetch, so an unreachable store does not hold it back.
name = "the hive collector's fetch revives a collector that hit its start limit without the file";
name = "the hive collector's fetch revives a collector that is failed without the file";
ok =
let
s = builtins.unsafeDiscardStringContext (hiveOtelFetch hiveOtelLeaf).script;
@ -396,7 +400,7 @@ let
# A secret delivered some other way is the operator's: no unit fetches
# into a file the collector does not load, or restarts it on one. Nothing
# restarts the collector when that file lands, so it waits for the file
# instead of failing `LoadCredential` into its start limit.
# instead of looping on a failing `LoadCredential`.
name = "a hand-named collector secret renders no store fetch, and the collector waits for the file";
ok =
let

33
nix/module-eval/swarm-otel-core.nix
View file

@ -267,25 +267,28 @@ let
ok = carriesJournaldSeverity (otelSettings otelNoStores).receivers.journald;
}
{
# Sibling of ./hive-otel.nix's identical case for the host-tier
# collector: this one's `oidc/*` extensions call out to authelia at
# startup, so it fails the same way when that restart races its own.
# nixpkgs ships `Restart = "always"` with no `RestartSec`, so without
# this the unit burns its five default attempts inside two seconds and
# lands in `start-limit-hit`, where it stops retrying. The third clause
# is the one that has to hold — `StartLimit*` are `[Unit]` settings
# that systemd ignores under `[Service]`, so a bound written into
# `serviceConfig` renders, deploys and does nothing.
name = "the swarm collector backs off a failed bind from [Unit], not [Service]";
# The `oidc/*` extensions fetch authelia's discovery document through
# the gateway at startup, and the collector exits 1 when that fails. Any
# start limit leaves it failed for good once the gateway has been away
# long enough, so the limit has to be off and the delay has to grow.
# `StartLimit*` are `[Unit]` settings that systemd ignores under
# `[Service]`, so a limit written into `serviceConfig` renders, deploys
# and does nothing. `Restart = "always"` is nixpkgs' own setting: it
# shows the lookup reaches the real unit, so a misspelt attribute path
# cannot pass the other clauses.
name = "the swarm collector retries a failed start forever, with a growing delay";
ok =
let
u = otelNoStores.containers.swarm-otel.config.systemd.services.opentelemetry-collector;
sc = u.serviceConfig;
in
u.serviceConfig.RestartSec or 0 > 0
&& toString u.unitConfig.StartLimitBurst == "12"
&& !(u.serviceConfig ? StartLimitBurst)
# The window has to outlast every attempt, or the burst is unreachable.
&& u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst;
u.serviceConfig.Restart == "always"
&& toString (u.unitConfig.StartLimitIntervalSec or "unset") == "0"
&& !(sc ? StartLimitIntervalSec)
&& !(sc ? StartLimitBurst)
&& sc.RestartSec > 0
&& sc.RestartSteps > 0
&& sc.RestartMaxDelaySec > sc.RestartSec;
}
{
# 🩸 The container's own journal reaches the stores from the collector