diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 9782f3ed..02e93162 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -1757,14 +1757,37 @@ in }; }; - # The credential file is already `NAME=value`, systemd's - # EnvironmentFile format — so the secret reaches the process as an - # environment variable without being read by nix, written to the - # store, or passed in argv. - systemd.services.opentelemetry-collector.serviceConfig = - lib.optionalAttrs (otelCfg.headersCredential != null) { - EnvironmentFile = otelCfg.headersCredential; + systemd.services.opentelemetry-collector = lib.mkMerge [ + # ⚠️ `StartLimit*` are `[Unit]` settings; systemd IGNORES them + # under `[Service]` — in `serviceConfig` they render, deploy and + # do nothing. `checks.module-eval-swarm-otel-core` pins them in + # `unitConfig`, same shape as `checks.module-eval-hive-otel` for + # the sibling host-tier collector. `Restart` is deliberately NOT + # set: nixpkgs' own module defines it at normal priority, so a + # second definition is a module-system conflict. + # + # This collector's `oidc/*` extensions call out to authelia at + # startup, so a start that races authelia's own restart fails — + # and with systemd's defaults (100ms `RestartSec`, 5-in-10s + # start limit) that burns the whole allowance before authelia is + # back, landing in `start-limit-hit`. The window here (12 × 5s = + # 60s) has to exceed `RestartSec × burst` with margin for that + # drift, same values as `otel.nix`'s host-tier collector, which + # shares the same dependency. + { + startLimitBurst = 12; + startLimitIntervalSec = 120; + serviceConfig.RestartSec = 5; } + + # The credential file is already `NAME=value`, systemd's + # EnvironmentFile format — so the secret reaches the process as + # an environment variable without being read by nix, written to + # the store, or passed in argv. + (lib.optionalAttrs (otelCfg.headersCredential != null) { + serviceConfig.EnvironmentFile = otelCfg.headersCredential; + }) + # The other credential, and the other direction: the one above # authenticates this collector's export onward, this one # authenticates it to a service it scrapes. @@ -1776,9 +1799,12 @@ in # exactly rather than restating a narrower one. Where it is false # no authenticator is rendered either, so the collector starts and # is refused by the stores rather than failing to start. - // lib.optionalAttrs haveCollectorSecret { - LoadCredential = [ "${collectorCredentialId}:${deployCfg.swarm-otel.clientSecretFile}" ]; - }; + (lib.optionalAttrs haveCollectorSecret { + serviceConfig.LoadCredential = [ + "${collectorCredentialId}:${deployCfg.swarm-otel.clientSecretFile}" + ]; + }) + ]; }; }; }; diff --git a/nix/module-eval/swarm-otel-core.nix b/nix/module-eval/swarm-otel-core.nix index dbf0a70a..1127b92f 100644 --- a/nix/module-eval/swarm-otel-core.nix +++ b/nix/module-eval/swarm-otel-core.nix @@ -223,6 +223,27 @@ let name = "the swarm collector's journald receiver carries the shared PRIORITY mapping"; ok = carriesJournaldSeverity (otelSettings otelNoStores).receivers.journald; } + { + # Sibling of ./hive-otel.nix's identical case for the host-tier + # collector: this one's `oidc/*` extensions call out to authelia at + # startup, so it fails the same way when that restart races its own. + # nixpkgs ships `Restart = "always"` with no `RestartSec`, so without + # this the unit burns its five default attempts inside two seconds and + # lands in `start-limit-hit`, where it stops retrying. The third clause + # is the one that has to hold — `StartLimit*` are `[Unit]` settings + # that systemd ignores under `[Service]`, so a bound written into + # `serviceConfig` renders, deploys and does nothing. + name = "the swarm collector backs off a failed bind from [Unit], not [Service]"; + ok = + let + u = otelNoStores.containers.swarm-otel.config.systemd.services.opentelemetry-collector; + in + u.serviceConfig.RestartSec or 0 > 0 + && toString u.unitConfig.StartLimitBurst == "12" + && !(u.serviceConfig ? StartLimitBurst) + # The window has to outlast every attempt, or the burst is unreachable. + && u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst; + } ]; in runGroup "swarm-otel-core" cases