diff --git a/docs/observability.md b/docs/observability.md index 35f22bf1..8a2d627d 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -18,7 +18,7 @@ services.hyperhive.otel = { `enable` is the single gate. `endpoint` is where telemetry ends up after it leaves the swarm — optional, because the swarm's own metrics store -(`swarm.victoriametrics.enable`) is a destination in its own right. With both, +(`deploy.victoriametrics`) is a destination in its own right. With both, telemetry goes to both. See [`swarm/services.md`](swarm/services.md#metrics-victoriametrics--grafana). diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 4668c9a3..996a634c 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -74,7 +74,7 @@ together: a store with no UI is unreadable and a UI with no store is empty. To run one without the other, set it directly: ```nix -services.hyperhive.swarm.victoriametrics.enable = true; +services.hyperhive.deploy.victoriametrics = true; services.hyperhive.deploy.grafana = false; ``` diff --git a/nix/host-modules/deploy.nix b/nix/host-modules/deploy.nix index 7fdc5d73..4a95720c 100644 --- a/nix/host-modules/deploy.nix +++ b/nix/host-modules/deploy.nix @@ -34,6 +34,14 @@ [ "services" "hyperhive" "swarm" "grafana" "enable" ] [ "services" "hyperhive" "deploy" "grafana" ] ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "victoriametrics" "enable" ] + [ "services" "hyperhive" "deploy" "victoriametrics" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "victorialogs" "enable" ] + [ "services" "hyperhive" "deploy" "victorialogs" ] + ) ]; options.services.hyperhive.deploy = { @@ -49,5 +57,33 @@ whether hyperhive is installed. ''; }; + + victoriametrics = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Run the swarm's metrics store on this host. + + Derives from + {option}`services.hyperhive.swarm.enableRequiredServices` together + with {option}`services.hyperhive.deploy.grafana`: a store with no + UI is unreadable and a UI with no store is empty, so there is no + sensible deployment that takes one and not the other from that + switch. Set either directly to run exactly one. + ''; + }; + + victorialogs = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Run the swarm's log store on this host. + + Derives from + {option}`services.hyperhive.swarm.enableRequiredServices` for the + same reason as the metrics pair above: a hive that is not the + service host is a *client* of this store, not a second one. + ''; + }; }; } diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 40b9e1b8..b1cc2d7b 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -87,7 +87,7 @@ in never given to a container. Optional. Leave it empty and the swarm's own metrics store - ({option}`services.hyperhive.swarm.victoriametrics.enable`) is the + ({option}`services.hyperhive.deploy.victoriametrics`) is the destination; that is a complete deployment, not a degraded one. Set both and telemetry goes to both. ''; diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index b39e66bd..e899ab24 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -33,6 +33,7 @@ let otelCfg = config.services.hyperhive.otel; vmCfg = config.services.hyperhive.swarm.victoriametrics; vlCfg = config.services.hyperhive.swarm.victorialogs; + deployCfg = config.services.hyperhive.deploy; hyperhiveCfg = config.services.hyperhive; gatewayCfg = hyperhiveCfg.gateway; swarmDomain = hyperhiveCfg.swarm.domain; @@ -162,13 +163,14 @@ let # exactly the same destinations as the single pipeline they replace. # Written once because "which exporters" is a property of this tier, not # of which hive a sample came from. - exporterNames = upstreamExporters ++ lib.optional vmCfg.enable "otlphttp/victoriametrics"; + exporterNames = + upstreamExporters ++ lib.optional deployCfg.victoriametrics "otlphttp/victoriametrics"; # The same fan-out for logs, and the local store is only ONE of its # destinations. A deployment that turns the swarm's log store off and keeps # an upstream endpoint still collects — the store is where logs may be kept, # not the reason to read the journal at all. - logExporterNames = upstreamExporters ++ lib.optional vlCfg.enable "otlphttp/victorialogs"; + logExporterNames = upstreamExporters ++ lib.optional deployCfg.victorialogs "otlphttp/victorialogs"; # Collect when there is anywhere to send it, and only then. A pipeline with # an empty exporter list is not a quiet no-op — the collector rejects it — @@ -608,12 +610,12 @@ in # The tier exists to hold the upstream credential and to write the # swarm's store. With neither, it is a process that receives # samples and drops them — which looks healthy and loses data. - assertion = otelCfg.endpoint != "" || vmCfg.enable; + assertion = otelCfg.endpoint != "" || deployCfg.victoriametrics; message = '' services.hyperhive.swarm.otel.enable is true but this collector has nowhere to send what it receives: services.hyperhive.otel.endpoint is empty and - services.hyperhive.swarm.victoriametrics.enable is false. + services.hyperhive.deploy.victoriametrics is false. Set the endpoint to export upstream, or enable the swarm's metrics store. @@ -780,7 +782,7 @@ in cfg.producerPort otelCfg.collector.port ] - ++ lib.optional vmCfg.enable vmCfg.port; + ++ lib.optional deployCfg.victoriametrics vmCfg.port; all = derived ++ others; in lib.length (lib.unique all) == lib.length all; @@ -1015,7 +1017,7 @@ in }; exporters = - lib.optionalAttrs vmCfg.enable { + lib.optionalAttrs deployCfg.victoriametrics { # `metrics_endpoint`, NOT `endpoint`: the latter is a # base that otlphttp appends `/v1/metrics` to, while # VictoriaMetrics serves OTLP at @@ -1038,7 +1040,7 @@ in } // lib.optionalAttrs (otelCfg.protocol == "http/json") { encoding = "json"; }; } - // lib.optionalAttrs vlCfg.enable { + // lib.optionalAttrs deployCfg.victorialogs { # `logs_endpoint`, NOT `endpoint`, for exactly the reason the # metrics exporter above spells out — and the trap is worse # here, because the two stores' OTLP routes differ. `endpoint` diff --git a/nix/host-modules/swarm-required-services.nix b/nix/host-modules/swarm-required-services.nix index dbaa26a0..78ebcc09 100644 --- a/nix/host-modules/swarm-required-services.nix +++ b/nix/host-modules/swarm-required-services.nix @@ -59,25 +59,10 @@ in # the queue they authenticate against. nats.enable = lib.mkDefault swarmCfg.enableRequiredServices; - # The metrics pair. Once per swarm and optional, so they meet the rule - # in the option's description the same way the three above do — a hive - # that is not the service host is a *client* of this Grafana, not a - # second one. - # - # They derive together on purpose: a store with no UI is unreadable and - # a UI with no store is empty, so there is no sensible deployment that - # takes one and not the other from this switch. An operator who wants - # exactly one still sets it directly, which `mkDefault` allows. - victoriametrics.enable = lib.mkDefault swarmCfg.enableRequiredServices; - # (grafana's half of the pair derives below — it lives in `deploy.*` - # now, which is a different attribute path, not a different rule.) - - # The log store, deriving from the same switch for the same reason — - # and deliberately in the same commit as the collector pipeline that - # writes to it, never before it. A store nothing writes to is worse - # than no store: it starts, answers queries, and returns nothing, so - # the first person to look concludes there were no logs. - victorialogs.enable = lib.mkDefault swarmCfg.enableRequiredServices; + # The metrics pair and the log store used to derive here too. They now + # live under `deploy.*` (below, and ./deploy.nix) because "does THIS + # host run it" is a per-host decision and `swarm.*` has to be identical + # on every host. Same switch, same rule, different attribute path. # The collector that feeds the pair above, and the only tier holding # the upstream credential. Same rule as the rest: once per swarm, @@ -90,9 +75,16 @@ in # this is ./otel.nix's existing per-hive option). config.services.hyperhive.otel.enable = lib.mkDefault swarmCfg.enableRequiredServices; - # Grafana, the UI half of the metrics pair. Same derivation and the same - # reasoning as `victoriametrics.enable` above; it reads differently only - # because "does this host run it" now lives in `deploy.*` (./deploy.nix) - # rather than under `swarm.*`, which has to be identical on every host. + # The metrics pair, deriving together on purpose: a store with no UI is + # unreadable and a UI with no store is empty, so there is no sensible + # deployment that takes one and not the other from this switch. An + # operator who wants exactly one still sets it directly, which + # `mkDefault` allows. + # + # The log store derives from the same switch for the same reason as the + # rest: a hive that is not the service host is a *client* of it, not a + # second one. + config.services.hyperhive.deploy.victoriametrics = lib.mkDefault swarmCfg.enableRequiredServices; + config.services.hyperhive.deploy.victorialogs = lib.mkDefault swarmCfg.enableRequiredServices; config.services.hyperhive.deploy.grafana = lib.mkDefault swarmCfg.enableRequiredServices; } diff --git a/nix/host-modules/swarm-victorialogs.nix b/nix/host-modules/swarm-victorialogs.nix index 68f87a1f..737f628e 100644 --- a/nix/host-modules/swarm-victorialogs.nix +++ b/nix/host-modules/swarm-victorialogs.nix @@ -29,6 +29,7 @@ }: let cfg = config.services.hyperhive.swarm.victorialogs; + deployCfg = config.services.hyperhive.deploy; networkCfg = config.services.hyperhive.network; hyperhiveCfg = config.services.hyperhive; gatewayCfg = hyperhiveCfg.gateway; @@ -52,18 +53,10 @@ let ''; in { + # `enable` moved to `services.hyperhive.deploy.victorialogs` — see + # ./deploy.nix. What stays here is what the store IS: its package, + # domain, retention and wiring. options.services.hyperhive.swarm.victorialogs = { - enable = lib.mkOption { - type = lib.types.bool; - default = false; - description = '' - Run the swarm's log store on this host. Off by default and not - derived from {option}`services.hyperhive.enable`: a swarm has one - log store, so enabling it is a decision about swarm topology - rather than about whether hyperhive is installed. - ''; - }; - package = lib.mkOption { type = lib.types.package; default = pkgs.victorialogs; @@ -128,7 +121,7 @@ in }; }; - config = lib.mkIf (hyperhiveCfg.enable && cfg.enable) { + config = lib.mkIf (hyperhiveCfg.enable && deployCfg.victorialogs) { # This store publishes its own health as prometheus metrics on the same # listener it serves queries on, so the swarm's collector scrapes it with # no exporter and no extra port — same arrangement as the metrics store. @@ -141,7 +134,7 @@ in # hosts are separate evaluations. services.hyperhive.swarm.otel.scrapeTargets.victorialogs = "127.0.0.1:${toString cfg.port}"; - # The gateway name and the quick-link, both inside `cfg.enable` — same + # The gateway name and the quick-link, both inside `deployCfg.victorialogs` — same # "only the host that runs the service may claim the name" guard every # sibling swarm-service module uses (`swarm-grafana.nix`, # `swarm-victoriametrics.nix`). diff --git a/nix/host-modules/swarm-victoriametrics.nix b/nix/host-modules/swarm-victoriametrics.nix index a991f989..c02def6d 100644 --- a/nix/host-modules/swarm-victoriametrics.nix +++ b/nix/host-modules/swarm-victoriametrics.nix @@ -17,6 +17,7 @@ }: let cfg = config.services.hyperhive.swarm.victoriametrics; + deployCfg = config.services.hyperhive.deploy; networkCfg = config.services.hyperhive.network; hyperhiveCfg = config.services.hyperhive; gatewayCfg = hyperhiveCfg.gateway; @@ -28,18 +29,10 @@ let domainBase = if swarmDomain == null then "invalid" else swarmDomain; in { + # `enable` moved to `services.hyperhive.deploy.victoriametrics` — see + # ./deploy.nix. What stays here is what the store IS: its package, + # domain, retention and wiring. options.services.hyperhive.swarm.victoriametrics = { - enable = lib.mkOption { - type = lib.types.bool; - default = false; - description = '' - Run the swarm's metrics store on this host. Off by default and not - derived from {option}`services.hyperhive.enable`: a swarm has one - metrics store, so enabling it is a decision about swarm topology - rather than about whether hyperhive is installed. - ''; - }; - package = lib.mkOption { type = lib.types.package; default = pkgs.victoriametrics; @@ -97,8 +90,8 @@ in }; }; - config = lib.mkIf (hyperhiveCfg.enable && cfg.enable) { - # The gateway name and the quick-link, both inside `cfg.enable` — that + config = lib.mkIf (hyperhiveCfg.enable && deployCfg.victoriametrics) { + # The gateway name and the quick-link, both inside `deployCfg.victoriametrics` — that # guard is the load-bearing part. Every hive in a swarm may know this # store exists, but only the host that RUNS it may claim the name; a # client hive declaring the vhost would answer for a service it does not diff --git a/nix/host-modules/swarm.nix b/nix/host-modules/swarm.nix index 6437b9fd..efe64565 100644 --- a/nix/host-modules/swarm.nix +++ b/nix/host-modules/swarm.nix @@ -59,7 +59,7 @@ let # them; the collector's exporter — same defect, no human in the loop — # failed every POST and dropped the samples. ++ lib.optional deployCfg.grafana swarmCfg.grafana.domain - ++ lib.optional swarmCfg.victoriametrics.enable swarmCfg.victoriametrics.domain + ++ lib.optional deployCfg.victoriametrics swarmCfg.victoriametrics.domain ++ lib.optional swarmCfg.otel.enable swarmCfg.otel.domain # VictoriaLogs' vhost is new (was previously unpublished entirely — see # swarm-victorialogs.nix's file-top comment) and needs the same @@ -67,7 +67,7 @@ let # this list, `gateway.lib.tlsFor` falls back to the hive leaf, which # cannot cover a name under a different apex — see the ⚠️ above this # list for what that looked like the last time a name was missed here. - ++ lib.optional swarmCfg.victorialogs.enable swarmCfg.victorialogs.domain; + ++ lib.optional deployCfg.victorialogs swarmCfg.victorialogs.domain; # Hives whose entry still carries the removed `certFingerprint`. Scanned # here, at top level, because that is the only place an assertion about a