diff --git a/docs/observability.md b/docs/observability.md index 8a2d627d..13b3d649 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -80,7 +80,7 @@ the host is: | | runs where | receives from | does | |---|---|---|---| | **hive tier** — `otel.enable` | every hive with agents | that hive's agents, on the bridge | forwards to the swarm tier. Holds no credential, picks no destination | -| **swarm tier** — `swarm.otel.enable` | once per swarm | every hive's collector | writes the swarm's store and exports upstream | +| **swarm tier** — `deploy.otel` | once per swarm | every hive's collector | writes the swarm's store and exports upstream | An all-local host runs both, and needs nothing said about the hop between them. diff --git a/nix/host-modules/deploy.nix b/nix/host-modules/deploy.nix index a58de18f..36204d58 100644 --- a/nix/host-modules/deploy.nix +++ b/nix/host-modules/deploy.nix @@ -61,6 +61,10 @@ in [ "services" "hyperhive" "swarm" "nats" "enable" ] [ "services" "hyperhive" "deploy" "nats" ] ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "otel" "enable" ] + [ "services" "hyperhive" "deploy" "otel" ] + ) ]; options.services.hyperhive.deploy = { @@ -123,6 +127,26 @@ in ''; }; + otel = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Run the **swarm's** telemetry collector on this host. + + Derives from + {option}`services.hyperhive.swarm.enableRequiredServices` with the + metrics pair it feeds: a swarm has one of these, and it belongs + wherever the shared services live rather than on every hive. + + ⚠️ Not to be confused with + {option}`services.hyperhive.otel.enable`, the **hive-tier** + collector, which every hive runs and which is a different option. + A hive that does not run the swarm collector still runs its own, + and reaches this one by name at + {option}`services.hyperhive.swarm.otel.domain`. + ''; + }; + nats = lib.mkOption { type = lib.types.bool; default = false; diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 56bfa2e7..0aa1c186 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -61,7 +61,7 @@ in their own containers can reach. That collector forwards to the swarm's - ({option}`services.hyperhive.swarm.otel.enable`), which holds the + ({option}`services.hyperhive.deploy.otel`), which holds the upstream credential and writes the swarm's store. So an agent never sees the credential, and neither does this tier. @@ -81,7 +81,7 @@ in has left the swarm. Read by the swarm's collector - ({option}`services.hyperhive.swarm.otel.enable`), which is the only + ({option}`services.hyperhive.deploy.otel`), which is the only tier that holds the upstream credential. An agent is handed the *first* hop instead — this hive's own collector — so this value is never given to a container. diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index a9ddf5b9..f4728f91 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -1009,7 +1009,7 @@ in # Written out in full rather than through a `let` binding so the gate and # the option it gates are visibly the same path: gating the wrong one is # not a build error, it is a target that is always declared or never is. - services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf config.services.hyperhive.swarm.otel.enable { + services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf config.services.hyperhive.deploy.otel { authelia = "127.0.0.1:${toString cfg.metricsPort}"; }; diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix index 4c61eb9c..fd0e7165 100644 --- a/nix/host-modules/swarm-controller.nix +++ b/nix/host-modules/swarm-controller.nix @@ -174,14 +174,14 @@ let # so the two ends agree on exactly one path, `/v1/metrics`, arrived at # from opposite directions. # - # Gated on `otelSwarmCfg.enable` alone (a swarm-wide fact, not "enabled + # Gated on `deployCfg.otel` alone (a swarm-wide fact, not "enabled # HERE"): the collector is reachable by name wherever it runs, so a # controller not co-located with it now exports over https instead of # exporting nothing — the graceful-absence case left is "no swarm-otel # anywhere in this swarm at all," which `forgeEnv` already models the # same way. otelSwarmCfg = config.services.hyperhive.swarm.otel; - otelEnv = lib.optionalAttrs otelSwarmCfg.enable { + otelEnv = lib.optionalAttrs deployCfg.otel { OTEL_EXPORTER_OTLP_ENDPOINT = "https://${otelSwarmCfg.domain}/${otelSwarmCfg.producerName}"; # Checked by `swarm-otel.nix`'s `oidc/${producerName}` authenticator # against exactly this controller's own registered audience (see the diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 02c7b931..ed18b7f2 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -408,7 +408,7 @@ in # Declared here rather than in the collector's module, per that option's # rule: an entry exists where the service that named it runs. - services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf config.services.hyperhive.swarm.otel.enable { + services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf config.services.hyperhive.deploy.otel { grafana = "127.0.0.1:${toString cfg.metricsPort}"; }; diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index 973f191f..8f9fb6a6 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -535,7 +535,7 @@ in # ⚠️ `swarm.otel`, not `hyperhive.otel` — two collectors one word apart, # and only this one reads `scrapeTargets`. Written in full so the gate # and the option it gates are visibly the same path. - services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf config.services.hyperhive.swarm.otel.enable { + services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf config.services.hyperhive.deploy.otel { nats = "127.0.0.1:${toString cfg.metricsPort}"; }; diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 5dbac7ea..c3ec4851 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -29,11 +29,11 @@ }: let cfg = config.services.hyperhive.swarm.otel; + deployCfg = config.services.hyperhive.deploy; swarmCfg = config.services.hyperhive.swarm; otelCfg = config.services.hyperhive.otel; vmCfg = config.services.hyperhive.swarm.victoriametrics; vlCfg = config.services.hyperhive.swarm.victorialogs; - deployCfg = config.services.hyperhive.deploy; hyperhiveCfg = config.services.hyperhive; gatewayCfg = hyperhiveCfg.gateway; swarmDomain = hyperhiveCfg.swarm.domain; @@ -179,23 +179,11 @@ let collectLogs = logExporterNames != [ ]; in { + # `enable` moved to `services.hyperhive.deploy.otel` — see ./deploy.nix. + # ⚠️ That is the SWARM collector. The per-hive one keeps its own + # `services.hyperhive.otel.enable` (./otel.nix) and is a different + # option entirely — every hive runs that one. options.services.hyperhive.swarm.otel = { - enable = lib.mkOption { - type = lib.types.bool; - default = false; - description = '' - Run the swarm's telemetry collector on this host. - - Asserted from `swarm.enableRequiredServices` in - ./swarm-required-services.nix, with the metrics pair this - collector feeds: a swarm has one of these, and it belongs - wherever the shared services live rather than on every hive. - - A hive that does not run it still runs its own hive-tier collector - (`services.hyperhive.otel.enable`) and reaches this one by name, at - {option}`services.hyperhive.swarm.otel.domain`. - ''; - }; machine = lib.mkOption { type = lib.types.str; @@ -446,8 +434,8 @@ in }; }; - config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) { - # The gateway name, inside `cfg.enable` — that guard is the load-bearing + config = lib.mkIf (config.services.hyperhive.enable && deployCfg.otel) { + # The gateway name, inside `deployCfg.otel` — that guard is the load-bearing # part. Every hive in a swarm may know this collector exists, but only # the host that RUNS it may claim the name; a client hive declaring the # vhost would answer for a service it does not have. @@ -612,7 +600,7 @@ in # samples and drops them — which looks healthy and loses data. assertion = otelCfg.endpoint != "" || deployCfg.victoriametrics; message = '' - services.hyperhive.swarm.otel.enable is true but this collector + services.hyperhive.deploy.otel is true but this collector has nowhere to send what it receives: services.hyperhive.otel.endpoint is empty and services.hyperhive.deploy.victoriametrics is false. @@ -644,7 +632,7 @@ in # collector would listen on nothing while looking configured. assertion = hyperhiveCfg.swarm.hives != { }; message = '' - services.hyperhive.swarm.otel.enable is true but + services.hyperhive.deploy.otel is true but services.hyperhive.swarm.hives is empty: ingest is authenticated per hive, so an empty roster means this collector accepts nothing from anyone. @@ -756,7 +744,7 @@ in # issuer URL evaluates cleanly and refuses every hive at runtime. assertion = autheliaCfg.url != null; message = '' - services.hyperhive.swarm.otel.enable is true but + services.hyperhive.deploy.otel is true but services.hyperhive.swarm.authelia.url is null: every hive authenticates to this collector as itself, and the token comes from the swarm's identity provider. diff --git a/nix/host-modules/swarm-required-services.nix b/nix/host-modules/swarm-required-services.nix index 545e4981..b514a56c 100644 --- a/nix/host-modules/swarm-required-services.nix +++ b/nix/host-modules/swarm-required-services.nix @@ -49,28 +49,13 @@ in # Same precedence reasoning as ./local-defaults.nix: fills in for an # operator who hasn't spoken, yields to one who has. config.services.hyperhive.swarm = { + # The last one still spelled `swarm.*.enable`. Everything else that + # used to derive here — the queue, the SSO provider, the collector, + # the metrics pair, the log store — now derives below under + # `deploy.*`, because "does THIS host run it" is a per-host decision + # and `swarm.*` has to be identical on every host. Same switch, same + # rule, different attribute path. matrix.enable = lib.mkDefault swarmCfg.enableRequiredServices; - # The queue. Added later than the two above and missed at the time — - # this file predates the `swarm-nats` container by nine days and had - # not been revisited since, so its absence was sequence rather than - # intent. It meets the rule in the option's own description exactly: - # once per swarm, and optional. - # - # The tell that it was an omission: `local-defaults.nix` already - # derives `nats.autoGenerateCallout` from the all-local mode, so that - # mode was minting the queue's callout nkeys and then never starting - # the queue they authenticate against. - - # The metrics pair and the log store used to derive here too. They now - # live under `deploy.*` (below, and ./deploy.nix) because "does THIS - # host run it" is a per-host decision and `swarm.*` has to be identical - # on every host. Same switch, same rule, different attribute path. - - # The collector that feeds the pair above, and the only tier holding - # the upstream credential. Same rule as the rest: once per swarm, - # optional, and a hive that is not the service host is a *client* of - # it (by name, `swarm.otel.domain`) rather than a second one. - otel.enable = lib.mkDefault swarmCfg.enableRequiredServices; }; # The collector that feeds the pair above (note: no `swarm.` prefix, @@ -91,6 +76,11 @@ in # The queue. Same rule: once per swarm, optional. config.services.hyperhive.deploy.nats = lib.mkDefault swarmCfg.enableRequiredServices; + # The swarm collector that feeds the metrics pair, and the only tier + # holding the upstream credential. ⚠️ NOT the per-hive collector below, + # which every hive runs. + config.services.hyperhive.deploy.otel = lib.mkDefault swarmCfg.enableRequiredServices; + # The metrics pair, deriving together on purpose: a store with no UI is # unreadable and a UI with no store is empty, so there is no sensible # deployment that takes one and not the other from this switch. An diff --git a/nix/host-modules/swarm.nix b/nix/host-modules/swarm.nix index 5e8d6a5b..29d0baac 100644 --- a/nix/host-modules/swarm.nix +++ b/nix/host-modules/swarm.nix @@ -60,7 +60,7 @@ let # failed every POST and dropped the samples. ++ lib.optional deployCfg.grafana swarmCfg.grafana.domain ++ lib.optional deployCfg.victoriametrics swarmCfg.victoriametrics.domain - ++ lib.optional swarmCfg.otel.enable swarmCfg.otel.domain + ++ lib.optional deployCfg.otel swarmCfg.otel.domain # VictoriaLogs' vhost is new (was previously unpublished entirely — see # swarm-victorialogs.nix's file-top comment) and needs the same # membership every gateway-published swarm service needs: absent from