From 596c0c17bcb233317c2e87853f6d4161c2bb0b1a Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 23 Sep 2026 12:55:56 +0200 Subject: [PATCH] otel: back the hive collector off a failed bind instead of burning its start limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every deploy on a hive host, the replacement opentelemetry-collector reaches bind() while the outgoing process still holds 127.0.0.1:8888 (its self-scrape endpoint). nixpkgs sets Restart = "always" with no RestartSec, so the unit spends its five default attempts in under two seconds, hits start-limit-hit and stops retrying — ~27s of telemetry blackout per deploy. RestartSec = 5 with a 12-attempt burst over a 120s window rides the race out instead: the blackout ends within one interval of the port coming free, and 55s of it being held is survivable where 2s was not. StartLimitBurst/StartLimitIntervalSec go at the systemd.services attr level, which NixOS renders into [Unit]; under serviceConfig systemd ignores them silently. module-eval-hive-otel asserts the placement. --- nix/host-modules/otel.nix | 65 +++++++++++++++++++++++++++-------- nix/module-eval/hive-otel.nix | 21 +++++++++++ 2 files changed, 72 insertions(+), 14 deletions(-) diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 9cc0745b..ca9ff7a5 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -626,21 +626,58 @@ in }; }; - # `LoadCredential` and not a copy-oneshot: this collector is a HOST - # unit, so there is no container boundary to cross and therefore no - # reason for a second on-disk copy of the secret. systemd hands it to - # the process in a private tmpfs and exports the directory, which is - # what the config above names. - systemd.services.opentelemetry-collector = lib.optionalAttrs senderAuth { - serviceConfig.LoadCredential = [ "${credName}:${otel.clientSecretFile}" ]; + systemd.services.opentelemetry-collector = lib.mkMerge [ + # Survive a restart that races the outgoing process off + # `telemetryPort`. Every deploy, the replacement collector has been + # reaching `bind()` while the old one still holds `127.0.0.1:8888`, + # and nixpkgs' `Restart = "always"` with no `RestartSec` turns that + # into the worst possible response: five retries inside two seconds, + # `start-limit-hit`, and no telemetry at all until something else + # pokes the unit. Measured at ~27s of blackout per deploy. + # + # 🔑 Same inversion the credential wait below records — `always` + # reads like it makes this self-healing and does the opposite. A + # FAST-failing unit exhausts its allowance before the thing it is + # waiting for (here, an exiting process letting go of a socket) can + # possibly have happened. Nothing here removes that race; it makes + # the unit outlast it instead. + # + # Sized for a socket held by a process already on its way out — + # seconds, not an unseal's hours. 5s ends the blackout within one + # interval of the port coming free; 12 attempts ride out 55s of it + # being held, twice the worst gap observed. `Restart` itself is + # deliberately NOT set, same as both container collectors: nixpkgs' + # own module defines it at the normal priority, so a second + # definition is a conflict the module system refuses. + # + # ⚠️ `StartLimit*` are `[Unit]` settings and systemd IGNORES them + # under `[Service]` — written into `serviceConfig` they render, + # deploy and do nothing. These two options are the `[Unit]` + # spelling; `checks.module-eval-hive-otel` asserts they land in + # `unitConfig` and not beside `RestartSec`. The window has to + # exceed `RestartSec × burst`. + { + startLimitBurst = 12; + startLimitIntervalSec = 120; + serviceConfig.RestartSec = 5; + } - # `requires`, not merely `after`: ordering without gating leaves the - # collector starting anyway and failing on its own `LoadCredential`, - # which is the failure the wait exists to prevent. Same reasoning - # `lib/hive-ca-trust.nix` records for its bundle consumers. - requires = [ "${secretWaitUnit}.service" ]; - after = [ "${secretWaitUnit}.service" ]; - }; + # `LoadCredential` and not a copy-oneshot: this collector is a HOST + # unit, so there is no container boundary to cross and therefore no + # reason for a second on-disk copy of the secret. systemd hands it to + # the process in a private tmpfs and exports the directory, which is + # what the config above names. + (lib.mkIf senderAuth { + serviceConfig.LoadCredential = [ "${credName}:${otel.clientSecretFile}" ]; + + # `requires`, not merely `after`: ordering without gating leaves the + # collector starting anyway and failing on its own `LoadCredential`, + # which is the failure the wait exists to prevent. Same reasoning + # `lib/hive-ca-trust.nix` records for its bundle consumers. + requires = [ "${secretWaitUnit}.service" ]; + after = [ "${secretWaitUnit}.service" ]; + }) + ]; # Wait for the credential rather than racing it. # diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix index 2d9c603d..656483ec 100644 --- a/nix/module-eval/hive-otel.nix +++ b/nix/module-eval/hive-otel.nix @@ -111,6 +111,27 @@ let in !(m ? address) && (lib.head m.readers).pull.exporter.prometheus.port == 8888; } + { + # A bind that loses the race with the outgoing process is the whole + # failure: nixpkgs ships `Restart = "always"` with no `RestartSec`, so + # without this the unit spends its five default attempts inside two + # seconds and lands in `start-limit-hit`, where it stops retrying. The + # third clause is the one that has to hold — `StartLimit*` are `[Unit]` + # settings that systemd ignores under `[Service]`, so a bound written + # into `serviceConfig` renders, deploys and does nothing. Asserted + # where nixpkgs puts it rather than where it was written, same as + # ./bao-grants.nix. + name = "the hive collector backs off a failed bind from [Unit], not [Service]"; + ok = + let + u = hiveOtel.systemd.services.opentelemetry-collector; + in + u.serviceConfig.RestartSec or 0 > 0 + && toString u.unitConfig.StartLimitBurst == "12" + && !(u.serviceConfig ? StartLimitBurst) + # The window has to outlast every attempt, or the burst is unreachable. + && u.startLimitIntervalSec or 0 > u.serviceConfig.RestartSec * u.startLimitBurst; + } { # Both collectors share a network namespace whenever they are # co-located, and this port appears in no config the port-collision