From ba5ab313b32f9e3477ee2512707cc664d0aa2673 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 30 Aug 2026 03:35:23 +0200 Subject: [PATCH] deploy: move the swarm collector toggle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The one move where the suffix grep is the wrong instrument. There are two otel options a word apart — `swarm.otel.enable` (one per swarm) and `otel.enable` (one per hive, every hive runs it) — so `\.otel\.enable` matches twenty-five references of which most must not change. The module already carries a comment warning about exactly this, on a line that names `swarm.otel` in full rather than through a binding. Triaged by hand and confirmed the other way round: after the move, a grep for the per-hive option still finds it in the files that should keep it. Also worth recording what the alias sweep cannot do. `swarm.nix` reads this as `swarmCfg.otel.enable`, where `swarmCfg = cfg.swarm` and `cfg = config.services.hyperhive` — an alias bound to an alias, two hops from the option path. No syntactic sweep resolves that chain, which is why the module system's own evaluation is the only complete check here and the static sweeps are a way to narrow the work, not to finish it. --- docs/observability.md | 2 +- nix/host-modules/deploy.nix | 24 +++++++++++++++ nix/host-modules/otel.nix | 4 +-- nix/host-modules/swarm-authelia.nix | 2 +- nix/host-modules/swarm-controller.nix | 4 +-- nix/host-modules/swarm-grafana.nix | 2 +- nix/host-modules/swarm-nats.nix | 2 +- nix/host-modules/swarm-otel.nix | 32 ++++++-------------- nix/host-modules/swarm-required-services.nix | 32 +++++++------------- nix/host-modules/swarm.nix | 2 +- 10 files changed, 54 insertions(+), 52 deletions(-) diff --git a/docs/observability.md b/docs/observability.md index 8a2d627d..13b3d649 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -80,7 +80,7 @@ the host is: | | runs where | receives from | does | |---|---|---|---| | **hive tier** — `otel.enable` | every hive with agents | that hive's agents, on the bridge | forwards to the swarm tier. Holds no credential, picks no destination | -| **swarm tier** — `swarm.otel.enable` | once per swarm | every hive's collector | writes the swarm's store and exports upstream | +| **swarm tier** — `deploy.otel` | once per swarm | every hive's collector | writes the swarm's store and exports upstream | An all-local host runs both, and needs nothing said about the hop between them. diff --git a/nix/host-modules/deploy.nix b/nix/host-modules/deploy.nix index a58de18f..36204d58 100644 --- a/nix/host-modules/deploy.nix +++ b/nix/host-modules/deploy.nix @@ -61,6 +61,10 @@ in [ "services" "hyperhive" "swarm" "nats" "enable" ] [ "services" "hyperhive" "deploy" "nats" ] ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "otel" "enable" ] + [ "services" "hyperhive" "deploy" "otel" ] + ) ]; options.services.hyperhive.deploy = { @@ -123,6 +127,26 @@ in ''; }; + otel = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Run the **swarm's** telemetry collector on this host. + + Derives from + {option}`services.hyperhive.swarm.enableRequiredServices` with the + metrics pair it feeds: a swarm has one of these, and it belongs + wherever the shared services live rather than on every hive. + + ⚠️ Not to be confused with + {option}`services.hyperhive.otel.enable`, the **hive-tier** + collector, which every hive runs and which is a different option. + A hive that does not run the swarm collector still runs its own, + and reaches this one by name at + {option}`services.hyperhive.swarm.otel.domain`. + ''; + }; + nats = lib.mkOption { type = lib.types.bool; default = false; diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 56bfa2e7..0aa1c186 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -61,7 +61,7 @@ in their own containers can reach. That collector forwards to the swarm's - ({option}`services.hyperhive.swarm.otel.enable`), which holds the + ({option}`services.hyperhive.deploy.otel`), which holds the upstream credential and writes the swarm's store. So an agent never sees the credential, and neither does this tier. @@ -81,7 +81,7 @@ in has left the swarm. Read by the swarm's collector - ({option}`services.hyperhive.swarm.otel.enable`), which is the only + ({option}`services.hyperhive.deploy.otel`), which is the only tier that holds the upstream credential. An agent is handed the *first* hop instead — this hive's own collector — so this value is never given to a container. diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index a9ddf5b9..f4728f91 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -1009,7 +1009,7 @@ in # Written out in full rather than through a `let` binding so the gate and # the option it gates are visibly the same path: gating the wrong one is # not a build error, it is a target that is always declared or never is. - services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf config.services.hyperhive.swarm.otel.enable { + services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf config.services.hyperhive.deploy.otel { authelia = "127.0.0.1:${toString cfg.metricsPort}"; }; diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix index 4c61eb9c..fd0e7165 100644 --- a/nix/host-modules/swarm-controller.nix +++ b/nix/host-modules/swarm-controller.nix @@ -174,14 +174,14 @@ let # so the two ends agree on exactly one path, `/v1/metrics`, arrived at # from opposite directions. # - # Gated on `otelSwarmCfg.enable` alone (a swarm-wide fact, not "enabled + # Gated on `deployCfg.otel` alone (a swarm-wide fact, not "enabled # HERE"): the collector is reachable by name wherever it runs, so a # controller not co-located with it now exports over https instead of # exporting nothing — the graceful-absence case left is "no swarm-otel # anywhere in this swarm at all," which `forgeEnv` already models the # same way. otelSwarmCfg = config.services.hyperhive.swarm.otel; - otelEnv = lib.optionalAttrs otelSwarmCfg.enable { + otelEnv = lib.optionalAttrs deployCfg.otel { OTEL_EXPORTER_OTLP_ENDPOINT = "https://${otelSwarmCfg.domain}/${otelSwarmCfg.producerName}"; # Checked by `swarm-otel.nix`'s `oidc/${producerName}` authenticator # against exactly this controller's own registered audience (see the diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 02c7b931..ed18b7f2 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -408,7 +408,7 @@ in # Declared here rather than in the collector's module, per that option's # rule: an entry exists where the service that named it runs. - services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf config.services.hyperhive.swarm.otel.enable { + services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf config.services.hyperhive.deploy.otel { grafana = "127.0.0.1:${toString cfg.metricsPort}"; }; diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index 973f191f..8f9fb6a6 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -535,7 +535,7 @@ in # ⚠️ `swarm.otel`, not `hyperhive.otel` — two collectors one word apart, # and only this one reads `scrapeTargets`. Written in full so the gate # and the option it gates are visibly the same path. - services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf config.services.hyperhive.swarm.otel.enable { + services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf config.services.hyperhive.deploy.otel { nats = "127.0.0.1:${toString cfg.metricsPort}"; }; diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 5dbac7ea..c3ec4851 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -29,11 +29,11 @@ }: let cfg = config.services.hyperhive.swarm.otel; + deployCfg = config.services.hyperhive.deploy; swarmCfg = config.services.hyperhive.swarm; otelCfg = config.services.hyperhive.otel; vmCfg = config.services.hyperhive.swarm.victoriametrics; vlCfg = config.services.hyperhive.swarm.victorialogs; - deployCfg = config.services.hyperhive.deploy; hyperhiveCfg = config.services.hyperhive; gatewayCfg = hyperhiveCfg.gateway; swarmDomain = hyperhiveCfg.swarm.domain; @@ -179,23 +179,11 @@ let collectLogs = logExporterNames != [ ]; in { + # `enable` moved to `services.hyperhive.deploy.otel` — see ./deploy.nix. + # ⚠️ That is the SWARM collector. The per-hive one keeps its own + # `services.hyperhive.otel.enable` (./otel.nix) and is a different + # option entirely — every hive runs that one. options.services.hyperhive.swarm.otel = { - enable = lib.mkOption { - type = lib.types.bool; - default = false; - description = '' - Run the swarm's telemetry collector on this host. - - Asserted from `swarm.enableRequiredServices` in - ./swarm-required-services.nix, with the metrics pair this - collector feeds: a swarm has one of these, and it belongs - wherever the shared services live rather than on every hive. - - A hive that does not run it still runs its own hive-tier collector - (`services.hyperhive.otel.enable`) and reaches this one by name, at - {option}`services.hyperhive.swarm.otel.domain`. - ''; - }; machine = lib.mkOption { type = lib.types.str; @@ -446,8 +434,8 @@ in }; }; - config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) { - # The gateway name, inside `cfg.enable` — that guard is the load-bearing + config = lib.mkIf (config.services.hyperhive.enable && deployCfg.otel) { + # The gateway name, inside `deployCfg.otel` — that guard is the load-bearing # part. Every hive in a swarm may know this collector exists, but only # the host that RUNS it may claim the name; a client hive declaring the # vhost would answer for a service it does not have. @@ -612,7 +600,7 @@ in # samples and drops them — which looks healthy and loses data. assertion = otelCfg.endpoint != "" || deployCfg.victoriametrics; message = '' - services.hyperhive.swarm.otel.enable is true but this collector + services.hyperhive.deploy.otel is true but this collector has nowhere to send what it receives: services.hyperhive.otel.endpoint is empty and services.hyperhive.deploy.victoriametrics is false. @@ -644,7 +632,7 @@ in # collector would listen on nothing while looking configured. assertion = hyperhiveCfg.swarm.hives != { }; message = '' - services.hyperhive.swarm.otel.enable is true but + services.hyperhive.deploy.otel is true but services.hyperhive.swarm.hives is empty: ingest is authenticated per hive, so an empty roster means this collector accepts nothing from anyone. @@ -756,7 +744,7 @@ in # issuer URL evaluates cleanly and refuses every hive at runtime. assertion = autheliaCfg.url != null; message = '' - services.hyperhive.swarm.otel.enable is true but + services.hyperhive.deploy.otel is true but services.hyperhive.swarm.authelia.url is null: every hive authenticates to this collector as itself, and the token comes from the swarm's identity provider. diff --git a/nix/host-modules/swarm-required-services.nix b/nix/host-modules/swarm-required-services.nix index 545e4981..b514a56c 100644 --- a/nix/host-modules/swarm-required-services.nix +++ b/nix/host-modules/swarm-required-services.nix @@ -49,28 +49,13 @@ in # Same precedence reasoning as ./local-defaults.nix: fills in for an # operator who hasn't spoken, yields to one who has. config.services.hyperhive.swarm = { + # The last one still spelled `swarm.*.enable`. Everything else that + # used to derive here — the queue, the SSO provider, the collector, + # the metrics pair, the log store — now derives below under + # `deploy.*`, because "does THIS host run it" is a per-host decision + # and `swarm.*` has to be identical on every host. Same switch, same + # rule, different attribute path. matrix.enable = lib.mkDefault swarmCfg.enableRequiredServices; - # The queue. Added later than the two above and missed at the time — - # this file predates the `swarm-nats` container by nine days and had - # not been revisited since, so its absence was sequence rather than - # intent. It meets the rule in the option's own description exactly: - # once per swarm, and optional. - # - # The tell that it was an omission: `local-defaults.nix` already - # derives `nats.autoGenerateCallout` from the all-local mode, so that - # mode was minting the queue's callout nkeys and then never starting - # the queue they authenticate against. - - # The metrics pair and the log store used to derive here too. They now - # live under `deploy.*` (below, and ./deploy.nix) because "does THIS - # host run it" is a per-host decision and `swarm.*` has to be identical - # on every host. Same switch, same rule, different attribute path. - - # The collector that feeds the pair above, and the only tier holding - # the upstream credential. Same rule as the rest: once per swarm, - # optional, and a hive that is not the service host is a *client* of - # it (by name, `swarm.otel.domain`) rather than a second one. - otel.enable = lib.mkDefault swarmCfg.enableRequiredServices; }; # The collector that feeds the pair above (note: no `swarm.` prefix, @@ -91,6 +76,11 @@ in # The queue. Same rule: once per swarm, optional. config.services.hyperhive.deploy.nats = lib.mkDefault swarmCfg.enableRequiredServices; + # The swarm collector that feeds the metrics pair, and the only tier + # holding the upstream credential. ⚠️ NOT the per-hive collector below, + # which every hive runs. + config.services.hyperhive.deploy.otel = lib.mkDefault swarmCfg.enableRequiredServices; + # The metrics pair, deriving together on purpose: a store with no UI is # unreadable and a UI with no store is empty, so there is no sensible # deployment that takes one and not the other from this switch. An diff --git a/nix/host-modules/swarm.nix b/nix/host-modules/swarm.nix index 5e8d6a5b..29d0baac 100644 --- a/nix/host-modules/swarm.nix +++ b/nix/host-modules/swarm.nix @@ -60,7 +60,7 @@ let # failed every POST and dropped the samples. ++ lib.optional deployCfg.grafana swarmCfg.grafana.domain ++ lib.optional deployCfg.victoriametrics swarmCfg.victoriametrics.domain - ++ lib.optional swarmCfg.otel.enable swarmCfg.otel.domain + ++ lib.optional deployCfg.otel swarmCfg.otel.domain # VictoriaLogs' vhost is new (was previously unpublished entirely — see # swarm-victorialogs.nix's file-top comment) and needs the same # membership every gateway-published swarm service needs: absent from