mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

collectors: retry a failed start forever with a growing delay instead of hitting start-limit

Browse source
This commit is contained in:
atlas 2026-11-11 00:42:27 +01:00 • committed by mara
commit d48cc83dcb
5 changed files with 101 additions and 74 deletions
Download patch file Download diff file Expand all files Collapse all files

12
docs/4-implementation/swarm/telemetry.md
View file

@ -231,6 +231,18 @@ the collector's 4xx included. A switch leaves a window of about 20 seconds in wh
nginx and the container are out of step: the container stops before nginx
restarts and starts about 18 seconds after it. Senders retry through that window.
The collector fetches authelia's discovery document through the gateway when
it starts and exits if that fails. systemd restarts it with no start limit and a
delay that grows from 2 to 60 seconds, so it comes back by itself once the
gateway and authelia answer. If ingest stops, `systemctl status
swarm-otel-sockets` on the host shows whether the receiver sockets are there;
`machinectl shell swarm-otel /run/current-system/sw/bin/journalctl -u
opentelemetry-collector` shows why the collector didn't start. A collector that
keeps retrying never reaches `failed`, so `systemctl --failed` doesn't list it.
The signals are `swarm-otel-sockets` failing and the Grafana alert
`no log records from <hive>`, which fires after about 15 minutes without a log
record from a hive.
Every hive's own collector reaches this one by its gateway name,
`swarm.otel.domain` (default `otel.<swarm domain>`) — the same
by-domain-through-the-gateway shape every other swarm service uses, not a

43
nix/host-modules/otel.nix
View file

@ -768,16 +768,26 @@ in
};
systemd.services.opentelemetry-collector = lib.mkMerge [
# ⚠️ `StartLimit*` are `[Unit]` settings; systemd IGNORES them under
# `[Service]` — in `serviceConfig` they render, deploy and do
# nothing. `checks.module-eval-hive-otel` pins them in `unitConfig`.
# `Restart` is deliberately NOT set: nixpkgs' own module defines it
# at normal priority, so a second definition is a module-system
# conflict. The window must exceed `RestartSec × burst`.
# A failed start (a bind that loses the race with the outgoing
# process, a credential not yet delivered) must keep retrying: any
# start limit leaves the collector failed, and the hive's telemetry
# queues on disk, until someone starts it by hand.
#
# ⚠️ `StartLimitIntervalSec` is a `[Unit]` setting; systemd IGNORES it
# under `[Service]`, so it is the unit-level option here and
# `checks.module-eval-hive-otel` pins it in `unitConfig`. `Restart` is
# deliberately NOT set: nixpkgs' own module defines it as `"always"` at
# normal priority, so a second definition is a module-system conflict.
#
# The delay grows from `RestartSec` to `RestartMaxDelaySec` over
# `RestartSteps` restarts (systemd.service(5)).
{
startLimitBurst = 12;
startLimitIntervalSec = 120;
serviceConfig.RestartSec = 5;
startLimitIntervalSec = 0;
serviceConfig = {
RestartSec = 2;
RestartSteps = 6;
RestartMaxDelaySec = 60;
};
}
# `LoadCredential` and not a copy-oneshot: this collector is a HOST
@ -803,17 +813,12 @@ in
#
# `LoadCredential` naming a missing path is FATAL AT UNIT START, and
# nothing orders this host unit against whatever delivers the file.
# Worse, nixpkgs
# sets `Restart = "always"` with no `RestartSec`, so the failure is
# instant and the unit burns systemd's 5-starts-in-10s allowance in well
# under a second, landing in `start-limit-hit` where it stops retrying
# entirely and needs `systemctl reset-failed` by hand.
# The collector's restart policy retries a failed start with no limit,
# so without this wait the unit loops on the missing file, one start a
# minute at worst, until the file lands.
#
# 🔑 `Restart = always` reads like it makes this self-healing and does
# the opposite: a SLOW-failing unit retries until the secret appears, a
# FAST-failing one exhausts its limit before the thing it waits for can
# exist. This oneshot converts the fast failure into a slow one, which
# is what that restart policy is actually good at.
# This oneshot holds the collector back until the file exists, so the
# first start succeeds instead of failing and backing off.
#
# Not rendered beside the store fetch below, which restarts the
# collector itself when the file lands.

41
nix/host-modules/swarm-otel.nix
View file

@ -730,7 +730,7 @@ in
fi
# `LoadCredential` copies the file at start only, and a collector that
# started without it is in its start limit.
# started without it is in its growing-delay retry loop.
refresh_consumer ${lib.escapeShellArg cfg.machine} opentelemetry-collector.service ${lib.escapeShellArg collectorSecretInContainer}
'';
};
@ -1745,26 +1745,29 @@ in
};
systemd.services.opentelemetry-collector = lib.mkMerge [
# ⚠️ `StartLimit*` are `[Unit]` settings; systemd IGNORES them
# under `[Service]` — in `serviceConfig` they render, deploy and
# do nothing. `checks.module-eval-swarm-otel-core` pins them in
# `unitConfig`, same shape as `checks.module-eval-hive-otel` for
# the sibling host-tier collector. `Restart` is deliberately NOT
# set: nixpkgs' own module defines it at normal priority, so a
# second definition is a module-system conflict.
# The `oidc/*` extensions fetch authelia's discovery document
# through the gateway at startup, and the collector exits 1 when
# that fails. A gateway or authelia that is briefly away must not
# end the retrying: any start limit leaves the unit failed, with no
# receiver sockets, until someone starts it by hand.
#
# This collector's `oidc/*` extensions call out to authelia at
# startup, so a start that races authelia's own restart fails —
# and with systemd's defaults (100ms `RestartSec`, 5-in-10s
# start limit) that burns the whole allowance before authelia is
# back, landing in `start-limit-hit`. The window here (12 × 5s =
# 60s) has to exceed `RestartSec × burst` with margin for that
# drift, same values as `otel.nix`'s host-tier collector, which
# shares the same dependency.
# ⚠️ `StartLimitIntervalSec` is a `[Unit]` setting; systemd IGNORES
# it under `[Service]`, so it is the unit-level option here and
# `checks.module-eval-swarm-otel-core` pins it in `unitConfig`.
# `Restart` is deliberately NOT set: nixpkgs' own module defines it
# as `"always"` at normal priority, so a second definition is a
# module-system conflict.
#
# The delay grows from `RestartSec` to `RestartMaxDelaySec` over
# `RestartSteps` restarts (systemd.service(5)): a long outage costs
# one start a minute.
{
startLimitBurst = 12;
startLimitIntervalSec = 120;
serviceConfig.RestartSec = 5;
startLimitIntervalSec = 0;
serviceConfig = {
RestartSec = 2;
RestartSteps = 6;
RestartMaxDelaySec = 60;
};
}
{

46
nix/module-eval/hive-otel.nix
View file

@ -190,25 +190,28 @@ let
!(m ? address) && (lib.head m.readers).pull.exporter.prometheus.port == 8888;
}
{
# A bind that loses the race with the outgoing process is the whole
# failure: nixpkgs ships `Restart = "always"` with no `RestartSec`, so
# without this the unit spends its five default attempts inside two
# seconds and lands in `start-limit-hit`, where it stops retrying. The
# third clause is the one that has to hold — `StartLimit*` are `[Unit]`
# settings that systemd ignores under `[Service]`, so a bound written
# into `serviceConfig` renders, deploys and does nothing. Asserted
# where nixpkgs puts it rather than where it was written, same as
# ./bao-grants.nix.
name = "the hive collector backs off a failed bind from [Unit], not [Service]";
# A failed start (a bind that loses the race with the outgoing process,
# a credential not yet delivered) must keep retrying: any start limit
# leaves the collector failed until someone starts it by hand.
# `StartLimit*` are `[Unit]` settings that systemd ignores under
# `[Service]`, so a limit written into `serviceConfig` renders, deploys
# and does nothing. Asserted where nixpkgs puts it rather than where it
# was written, same as ./bao-grants.nix. `Restart = "always"` is
# nixpkgs' own setting: it shows the lookup reaches the real unit, so a
# misspelt attribute path cannot pass the other clauses.
name = "the hive collector retries a failed start forever, with a growing delay";
ok =
let
u = hiveOtel.systemd.services.opentelemetry-collector;
sc = u.serviceConfig;
in
u.serviceConfig.RestartSec or 0 > 0
&& toString u.unitConfig.StartLimitBurst == "12"
&& !(u.serviceConfig ? StartLimitBurst)
# The window has to outlast every attempt, or the burst is unreachable.
&& u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst;
sc.Restart == "always"
&& toString (u.unitConfig.StartLimitIntervalSec or "unset") == "0"
&& !(sc ? StartLimitIntervalSec)
&& !(sc ? StartLimitBurst)
&& sc.RestartSec > 0
&& sc.RestartSteps > 0
&& sc.RestartMaxDelaySec > sc.RestartSec;
}
{
# Collectors share a network namespace whenever they are co-located,
@ -319,14 +322,15 @@ let
&& lib.hasInfix ''if [ "$written_us" -le "$started_us" ]; then'' s;
}
{
# On a first boot the file does not exist until the fetch succeeds, so
# the collector fails `LoadCredential` into its start limit, i.e. the
# `failed` state. Only `reset-failed` clears that limit, so the fetch
# has to reset before it starts; `try-restart` acts on no failed unit.
# On a first boot the file does not exist until the fetch succeeds, and
# the collector retries `LoadCredential` with a growing delay. A
# collector that is `failed` for another reason is not retried by
# systemd, and `try-restart` acts on no failed unit, so the fetch has to
# `reset-failed` before it starts.
# A never-active collector has an empty `ActiveEnterTimestamp`, read as
# 0, so the file always counts as newer. The collector only `wants` the
# fetch, so an unreachable store does not hold it back.
name = "the hive collector's fetch revives a collector that hit its start limit without the file";
name = "the hive collector's fetch revives a collector that is failed without the file";
ok =
let
s = builtins.unsafeDiscardStringContext (hiveOtelFetch hiveOtelLeaf).script;
@ -396,7 +400,7 @@ let
# A secret delivered some other way is the operator's: no unit fetches
# into a file the collector does not load, or restarts it on one. Nothing
# restarts the collector when that file lands, so it waits for the file
# instead of failing `LoadCredential` into its start limit.
# instead of looping on a failing `LoadCredential`.
name = "a hand-named collector secret renders no store fetch, and the collector waits for the file";
ok =
let

33
nix/module-eval/swarm-otel-core.nix
View file

@ -267,25 +267,28 @@ let
ok = carriesJournaldSeverity (otelSettings otelNoStores).receivers.journald;
}
{
# Sibling of ./hive-otel.nix's identical case for the host-tier
# collector: this one's `oidc/*` extensions call out to authelia at
# startup, so it fails the same way when that restart races its own.
# nixpkgs ships `Restart = "always"` with no `RestartSec`, so without
# this the unit burns its five default attempts inside two seconds and
# lands in `start-limit-hit`, where it stops retrying. The third clause
# is the one that has to hold — `StartLimit*` are `[Unit]` settings
# that systemd ignores under `[Service]`, so a bound written into
# `serviceConfig` renders, deploys and does nothing.
name = "the swarm collector backs off a failed bind from [Unit], not [Service]";
# The `oidc/*` extensions fetch authelia's discovery document through
# the gateway at startup, and the collector exits 1 when that fails. Any
# start limit leaves it failed for good once the gateway has been away
# long enough, so the limit has to be off and the delay has to grow.
# `StartLimit*` are `[Unit]` settings that systemd ignores under
# `[Service]`, so a limit written into `serviceConfig` renders, deploys
# and does nothing. `Restart = "always"` is nixpkgs' own setting: it
# shows the lookup reaches the real unit, so a misspelt attribute path
# cannot pass the other clauses.
name = "the swarm collector retries a failed start forever, with a growing delay";
ok =
let
u = otelNoStores.containers.swarm-otel.config.systemd.services.opentelemetry-collector;
sc = u.serviceConfig;
in
u.serviceConfig.RestartSec or 0 > 0
&& toString u.unitConfig.StartLimitBurst == "12"
&& !(u.serviceConfig ? StartLimitBurst)
# The window has to outlast every attempt, or the burst is unreachable.
&& u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst;
u.serviceConfig.Restart == "always"
&& toString (u.unitConfig.StartLimitIntervalSec or "unset") == "0"
&& !(sc ? StartLimitIntervalSec)
&& !(sc ? StartLimitBurst)
&& sc.RestartSec > 0
&& sc.RestartSteps > 0
&& sc.RestartMaxDelaySec > sc.RestartSec;
}
{
# 🩸 The container's own journal reaches the stores from the collector