diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 9cc0745b..84e6a81a 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -626,21 +626,35 @@ in }; }; - # `LoadCredential` and not a copy-oneshot: this collector is a HOST - # unit, so there is no container boundary to cross and therefore no - # reason for a second on-disk copy of the secret. systemd hands it to - # the process in a private tmpfs and exports the directory, which is - # what the config above names. - systemd.services.opentelemetry-collector = lib.optionalAttrs senderAuth { - serviceConfig.LoadCredential = [ "${credName}:${otel.clientSecretFile}" ]; + systemd.services.opentelemetry-collector = lib.mkMerge [ + # ⚠️ `StartLimit*` are `[Unit]` settings; systemd IGNORES them under + # `[Service]` — in `serviceConfig` they render, deploy and do + # nothing. `checks.module-eval-hive-otel` pins them in `unitConfig`. + # `Restart` is deliberately NOT set: nixpkgs' own module defines it + # at normal priority, so a second definition is a module-system + # conflict. The window must exceed `RestartSec × burst`. + { + startLimitBurst = 12; + startLimitIntervalSec = 120; + serviceConfig.RestartSec = 5; + } - # `requires`, not merely `after`: ordering without gating leaves the - # collector starting anyway and failing on its own `LoadCredential`, - # which is the failure the wait exists to prevent. Same reasoning - # `lib/hive-ca-trust.nix` records for its bundle consumers. - requires = [ "${secretWaitUnit}.service" ]; - after = [ "${secretWaitUnit}.service" ]; - }; + # `LoadCredential` and not a copy-oneshot: this collector is a HOST + # unit, so there is no container boundary to cross and therefore no + # reason for a second on-disk copy of the secret. systemd hands it to + # the process in a private tmpfs and exports the directory, which is + # what the config above names. + (lib.mkIf senderAuth { + serviceConfig.LoadCredential = [ "${credName}:${otel.clientSecretFile}" ]; + + # `requires`, not merely `after`: ordering without gating leaves the + # collector starting anyway and failing on its own `LoadCredential`, + # which is the failure the wait exists to prevent. Same reasoning + # `lib/hive-ca-trust.nix` records for its bundle consumers. + requires = [ "${secretWaitUnit}.service" ]; + after = [ "${secretWaitUnit}.service" ]; + }) + ]; # Wait for the credential rather than racing it. # diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix index 2d9c603d..656483ec 100644 --- a/nix/module-eval/hive-otel.nix +++ b/nix/module-eval/hive-otel.nix @@ -111,6 +111,27 @@ let in !(m ? address) && (lib.head m.readers).pull.exporter.prometheus.port == 8888; } + { + # A bind that loses the race with the outgoing process is the whole + # failure: nixpkgs ships `Restart = "always"` with no `RestartSec`, so + # without this the unit spends its five default attempts inside two + # seconds and lands in `start-limit-hit`, where it stops retrying. The + # third clause is the one that has to hold — `StartLimit*` are `[Unit]` + # settings that systemd ignores under `[Service]`, so a bound written + # into `serviceConfig` renders, deploys and does nothing. Asserted + # where nixpkgs puts it rather than where it was written, same as + # ./bao-grants.nix. + name = "the hive collector backs off a failed bind from [Unit], not [Service]"; + ok = + let + u = hiveOtel.systemd.services.opentelemetry-collector; + in + u.serviceConfig.RestartSec or 0 > 0 + && toString u.unitConfig.StartLimitBurst == "12" + && !(u.serviceConfig ? StartLimitBurst) + # The window has to outlast every attempt, or the burst is unreachable. + && u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst; + } { # Both collectors share a network namespace whenever they are # co-located, and this port appears in no config the port-collision