diff --git a/docs/4-implementation/swarm/telemetry.md b/docs/4-implementation/swarm/telemetry.md index 5652530f1..de6d3acba 100644 --- a/docs/4-implementation/swarm/telemetry.md +++ b/docs/4-implementation/swarm/telemetry.md @@ -231,6 +231,18 @@ the collector's 4xx included. A switch leaves a window of about 20 seconds in wh nginx and the container are out of step: the container stops before nginx restarts and starts about 18 seconds after it. Senders retry through that window. +The collector fetches authelia's discovery document through the gateway when +it starts and exits if that fails. systemd restarts it with no start limit and a +delay that grows from 2 to 60 seconds, so it comes back by itself once the +gateway and authelia answer. If ingest stops, `systemctl status +swarm-otel-sockets` on the host shows whether the receiver sockets are there; +`machinectl shell swarm-otel /run/current-system/sw/bin/journalctl -u +opentelemetry-collector` shows why the collector didn't start. A collector that +keeps retrying never reaches `failed`, so `systemctl --failed` doesn't list it. +The signals are `swarm-otel-sockets` failing and the Grafana alert +`no log records from `, which fires after about 15 minutes without a log +record from a hive. + Every hive's own collector reaches this one by its gateway name, `swarm.otel.domain` (default `otel.`) — the same by-domain-through-the-gateway shape every other swarm service uses, not a diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index bfe7da4dd..e902b48d0 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -768,16 +768,26 @@ in }; systemd.services.opentelemetry-collector = lib.mkMerge [ - # ⚠️ `StartLimit*` are `[Unit]` settings; systemd IGNORES them under - # `[Service]` — in `serviceConfig` they render, deploy and do - # nothing. `checks.module-eval-hive-otel` pins them in `unitConfig`. - # `Restart` is deliberately NOT set: nixpkgs' own module defines it - # at normal priority, so a second definition is a module-system - # conflict. The window must exceed `RestartSec × burst`. + # A failed start (a bind that loses the race with the outgoing + # process, a credential not yet delivered) must keep retrying: any + # start limit leaves the collector failed, and the hive's telemetry + # queues on disk, until someone starts it by hand. + # + # ⚠️ `StartLimitIntervalSec` is a `[Unit]` setting; systemd IGNORES it + # under `[Service]`, so it is the unit-level option here and + # `checks.module-eval-hive-otel` pins it in `unitConfig`. `Restart` is + # deliberately NOT set: nixpkgs' own module defines it as `"always"` at + # normal priority, so a second definition is a module-system conflict. + # + # The delay grows from `RestartSec` to `RestartMaxDelaySec` over + # `RestartSteps` restarts (systemd.service(5)). { - startLimitBurst = 12; - startLimitIntervalSec = 120; - serviceConfig.RestartSec = 5; + startLimitIntervalSec = 0; + serviceConfig = { + RestartSec = 2; + RestartSteps = 6; + RestartMaxDelaySec = 60; + }; } # `LoadCredential` and not a copy-oneshot: this collector is a HOST @@ -803,17 +813,12 @@ in # # `LoadCredential` naming a missing path is FATAL AT UNIT START, and # nothing orders this host unit against whatever delivers the file. - # Worse, nixpkgs - # sets `Restart = "always"` with no `RestartSec`, so the failure is - # instant and the unit burns systemd's 5-starts-in-10s allowance in well - # under a second, landing in `start-limit-hit` where it stops retrying - # entirely and needs `systemctl reset-failed` by hand. + # The collector's restart policy retries a failed start with no limit, + # so without this wait the unit loops on the missing file, one start a + # minute at worst, until the file lands. # - # 🔑 `Restart = always` reads like it makes this self-healing and does - # the opposite: a SLOW-failing unit retries until the secret appears, a - # FAST-failing one exhausts its limit before the thing it waits for can - # exist. This oneshot converts the fast failure into a slow one, which - # is what that restart policy is actually good at. + # This oneshot holds the collector back until the file exists, so the + # first start succeeds instead of failing and backing off. # # Not rendered beside the store fetch below, which restarts the # collector itself when the file lands. diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 37a4b2e77..255b64c2a 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -730,7 +730,7 @@ in fi # `LoadCredential` copies the file at start only, and a collector that - # started without it is in its start limit. + # started without it is in its growing-delay retry loop. refresh_consumer ${lib.escapeShellArg cfg.machine} opentelemetry-collector.service ${lib.escapeShellArg collectorSecretInContainer} ''; }; @@ -1745,26 +1745,29 @@ in }; systemd.services.opentelemetry-collector = lib.mkMerge [ - # ⚠️ `StartLimit*` are `[Unit]` settings; systemd IGNORES them - # under `[Service]` — in `serviceConfig` they render, deploy and - # do nothing. `checks.module-eval-swarm-otel-core` pins them in - # `unitConfig`, same shape as `checks.module-eval-hive-otel` for - # the sibling host-tier collector. `Restart` is deliberately NOT - # set: nixpkgs' own module defines it at normal priority, so a - # second definition is a module-system conflict. + # The `oidc/*` extensions fetch authelia's discovery document + # through the gateway at startup, and the collector exits 1 when + # that fails. A gateway or authelia that is briefly away must not + # end the retrying: any start limit leaves the unit failed, with no + # receiver sockets, until someone starts it by hand. # - # This collector's `oidc/*` extensions call out to authelia at - # startup, so a start that races authelia's own restart fails — - # and with systemd's defaults (100ms `RestartSec`, 5-in-10s - # start limit) that burns the whole allowance before authelia is - # back, landing in `start-limit-hit`. The window here (12 × 5s = - # 60s) has to exceed `RestartSec × burst` with margin for that - # drift, same values as `otel.nix`'s host-tier collector, which - # shares the same dependency. + # ⚠️ `StartLimitIntervalSec` is a `[Unit]` setting; systemd IGNORES + # it under `[Service]`, so it is the unit-level option here and + # `checks.module-eval-swarm-otel-core` pins it in `unitConfig`. + # `Restart` is deliberately NOT set: nixpkgs' own module defines it + # as `"always"` at normal priority, so a second definition is a + # module-system conflict. + # + # The delay grows from `RestartSec` to `RestartMaxDelaySec` over + # `RestartSteps` restarts (systemd.service(5)): a long outage costs + # one start a minute. { - startLimitBurst = 12; - startLimitIntervalSec = 120; - serviceConfig.RestartSec = 5; + startLimitIntervalSec = 0; + serviceConfig = { + RestartSec = 2; + RestartSteps = 6; + RestartMaxDelaySec = 60; + }; } { diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix index d01be0835..2a275ce1b 100644 --- a/nix/module-eval/hive-otel.nix +++ b/nix/module-eval/hive-otel.nix @@ -190,25 +190,28 @@ let !(m ? address) && (lib.head m.readers).pull.exporter.prometheus.port == 8888; } { - # A bind that loses the race with the outgoing process is the whole - # failure: nixpkgs ships `Restart = "always"` with no `RestartSec`, so - # without this the unit spends its five default attempts inside two - # seconds and lands in `start-limit-hit`, where it stops retrying. The - # third clause is the one that has to hold — `StartLimit*` are `[Unit]` - # settings that systemd ignores under `[Service]`, so a bound written - # into `serviceConfig` renders, deploys and does nothing. Asserted - # where nixpkgs puts it rather than where it was written, same as - # ./bao-grants.nix. - name = "the hive collector backs off a failed bind from [Unit], not [Service]"; + # A failed start (a bind that loses the race with the outgoing process, + # a credential not yet delivered) must keep retrying: any start limit + # leaves the collector failed until someone starts it by hand. + # `StartLimit*` are `[Unit]` settings that systemd ignores under + # `[Service]`, so a limit written into `serviceConfig` renders, deploys + # and does nothing. Asserted where nixpkgs puts it rather than where it + # was written, same as ./bao-grants.nix. `Restart = "always"` is + # nixpkgs' own setting: it shows the lookup reaches the real unit, so a + # misspelt attribute path cannot pass the other clauses. + name = "the hive collector retries a failed start forever, with a growing delay"; ok = let u = hiveOtel.systemd.services.opentelemetry-collector; + sc = u.serviceConfig; in - u.serviceConfig.RestartSec or 0 > 0 - && toString u.unitConfig.StartLimitBurst == "12" - && !(u.serviceConfig ? StartLimitBurst) - # The window has to outlast every attempt, or the burst is unreachable. - && u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst; + sc.Restart == "always" + && toString (u.unitConfig.StartLimitIntervalSec or "unset") == "0" + && !(sc ? StartLimitIntervalSec) + && !(sc ? StartLimitBurst) + && sc.RestartSec > 0 + && sc.RestartSteps > 0 + && sc.RestartMaxDelaySec > sc.RestartSec; } { # Collectors share a network namespace whenever they are co-located, @@ -319,14 +322,15 @@ let && lib.hasInfix ''if [ "$written_us" -le "$started_us" ]; then'' s; } { - # On a first boot the file does not exist until the fetch succeeds, so - # the collector fails `LoadCredential` into its start limit, i.e. the - # `failed` state. Only `reset-failed` clears that limit, so the fetch - # has to reset before it starts; `try-restart` acts on no failed unit. + # On a first boot the file does not exist until the fetch succeeds, and + # the collector retries `LoadCredential` with a growing delay. A + # collector that is `failed` for another reason is not retried by + # systemd, and `try-restart` acts on no failed unit, so the fetch has to + # `reset-failed` before it starts. # A never-active collector has an empty `ActiveEnterTimestamp`, read as # 0, so the file always counts as newer. The collector only `wants` the # fetch, so an unreachable store does not hold it back. - name = "the hive collector's fetch revives a collector that hit its start limit without the file"; + name = "the hive collector's fetch revives a collector that is failed without the file"; ok = let s = builtins.unsafeDiscardStringContext (hiveOtelFetch hiveOtelLeaf).script; @@ -396,7 +400,7 @@ let # A secret delivered some other way is the operator's: no unit fetches # into a file the collector does not load, or restarts it on one. Nothing # restarts the collector when that file lands, so it waits for the file - # instead of failing `LoadCredential` into its start limit. + # instead of looping on a failing `LoadCredential`. name = "a hand-named collector secret renders no store fetch, and the collector waits for the file"; ok = let diff --git a/nix/module-eval/swarm-otel-core.nix b/nix/module-eval/swarm-otel-core.nix index b9af6fd97..d119875b9 100644 --- a/nix/module-eval/swarm-otel-core.nix +++ b/nix/module-eval/swarm-otel-core.nix @@ -267,25 +267,28 @@ let ok = carriesJournaldSeverity (otelSettings otelNoStores).receivers.journald; } { - # Sibling of ./hive-otel.nix's identical case for the host-tier - # collector: this one's `oidc/*` extensions call out to authelia at - # startup, so it fails the same way when that restart races its own. - # nixpkgs ships `Restart = "always"` with no `RestartSec`, so without - # this the unit burns its five default attempts inside two seconds and - # lands in `start-limit-hit`, where it stops retrying. The third clause - # is the one that has to hold — `StartLimit*` are `[Unit]` settings - # that systemd ignores under `[Service]`, so a bound written into - # `serviceConfig` renders, deploys and does nothing. - name = "the swarm collector backs off a failed bind from [Unit], not [Service]"; + # The `oidc/*` extensions fetch authelia's discovery document through + # the gateway at startup, and the collector exits 1 when that fails. Any + # start limit leaves it failed for good once the gateway has been away + # long enough, so the limit has to be off and the delay has to grow. + # `StartLimit*` are `[Unit]` settings that systemd ignores under + # `[Service]`, so a limit written into `serviceConfig` renders, deploys + # and does nothing. `Restart = "always"` is nixpkgs' own setting: it + # shows the lookup reaches the real unit, so a misspelt attribute path + # cannot pass the other clauses. + name = "the swarm collector retries a failed start forever, with a growing delay"; ok = let u = otelNoStores.containers.swarm-otel.config.systemd.services.opentelemetry-collector; + sc = u.serviceConfig; in - u.serviceConfig.RestartSec or 0 > 0 - && toString u.unitConfig.StartLimitBurst == "12" - && !(u.serviceConfig ? StartLimitBurst) - # The window has to outlast every attempt, or the burst is unreachable. - && u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst; + u.serviceConfig.Restart == "always" + && toString (u.unitConfig.StartLimitIntervalSec or "unset") == "0" + && !(sc ? StartLimitIntervalSec) + && !(sc ? StartLimitBurst) + && sc.RestartSec > 0 + && sc.RestartSteps > 0 + && sc.RestartMaxDelaySec > sc.RestartSec; } { # 🩸 The container's own journal reaches the stores from the collector