From c364d262e59a798c08b9bc65f34594d0a797bf87 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 16 Aug 2026 21:54:56 +0200 Subject: [PATCH] feat(#3265): feed the store from the collector, and derive the pair MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback: the metrics pair had nothing writing into it, and it sat outside the switch that turns on every other swarm-wide service. The collector now exports to VictoriaMetrics as well as upstream — a fan-out, not a choice: a local store is for looking at this swarm, an upstream is for whoever aggregates across swarms. That makes a local store a complete destination on its own, so `otel.endpoint` is no longer required when it runs here; a hive with neither is still refused. The assertion only ever relaxes, so every config that evaluated before still does. `enableRequiredServices` now derives both halves, alongside matrix, authelia and nats. They derive together because a store with no UI is unreadable and a UI with no store is empty. --- docs/swarm/services.md | 28 ++++++---- nix/host-modules/otel.nix | 55 ++++++++++++++++++-- nix/host-modules/swarm-required-services.nix | 12 +++++ 3 files changed, 82 insertions(+), 13 deletions(-) diff --git a/docs/swarm/services.md b/docs/swarm/services.md index a95aeacd..8608cb6c 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -68,15 +68,19 @@ one Grafana, in two containers at `metrics.` and `grafana.`. Two containers rather than one so Grafana can be restarted or broken without taking the time-series database with it. -Both are **opt-in** — unlike authelia and matrix they do not follow -`swarm.enableRequiredServices`, because turning them on starts a -database that grows for as long as the swarm runs: +Both follow `swarm.enableRequiredServices` like authelia and matrix, so +the swarm's service host gets them with everything else. They derive +together: a store with no UI is unreadable and a UI with no store is +empty. To run one without the other, set it directly: ```nix services.hyperhive.swarm.victoriametrics.enable = true; -services.hyperhive.swarm.grafana.enable = true; +services.hyperhive.swarm.grafana.enable = false; ``` +⚠️ **This starts a database that grows for as long as the swarm runs.** +See `retentionPeriod` below before leaving it at its default. + | Option | When you'd touch it | |---|---| | `swarm.victoriametrics.retentionPeriod` | Default `5y`. Lower it once you have measured how fast this swarm actually fills a disk — the default is deliberately generous because too-short silently discards history you cannot get back. | @@ -89,9 +93,15 @@ login form is switched off whenever SSO is configured. If you enable Grafana on a host with no authelia, the form stays on and Grafana's default `admin`/`admin` applies; change it before exposing that host. -The metrics **arrive** from the swarm's OTEL collector, not from agents -directly — see [`../observability.md`](../observability.md). Neither -container is reachable except through the gateway: both bind loopback, -and VictoriaMetrics' write endpoint takes no credential, so the -collector is the only intended writer. +**Where the data comes from.** With `otel.enable` on, the hive's OTEL +collector writes into this store as well as to any upstream endpoint — +both, not one or the other, since a local store is for looking at this +swarm and an upstream is for whoever aggregates across swarms. That also +means `otel.endpoint` is no longer required when the store runs here: a +hive with a local store already has somewhere for telemetry to go. See +[`../observability.md`](../observability.md). + +Neither container is reachable except through the gateway: both bind +loopback, and VictoriaMetrics' write endpoint takes no credential, so +the collector is the only intended writer. diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index f19906fb..ca159c84 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -158,8 +158,25 @@ (lib.mkIf config.services.hyperhive.c0re.enable { assertions = lib.optionals config.services.hyperhive.otel.enable [ { - assertion = config.services.hyperhive.otel.endpoint != ""; - message = "services.hyperhive.otel.enable is true but services.hyperhive.otel.endpoint is empty."; + # Telemetry has to go SOMEWHERE, but "somewhere" stopped meaning + # "an upstream endpoint" once the swarm grew its own store: a hive + # running `swarm.victoriametrics` is a complete destination on its + # own, and requiring an external endpoint as well would make the + # all-local mode impossible to express. + # + # This only ever relaxes the old rule — every config that passed + # before still passes. + assertion = + config.services.hyperhive.otel.endpoint != "" + || config.services.hyperhive.swarm.victoriametrics.enable; + message = '' + services.hyperhive.otel.enable is true but telemetry has nowhere + to go: services.hyperhive.otel.endpoint is empty and + services.hyperhive.swarm.victoriametrics.enable is false. + + Set the endpoint to export upstream, or enable the swarm's + metrics store to keep telemetry on this host. + ''; } ]; }) @@ -176,6 +193,29 @@ # becoming "how agents talk to the collector". The agent half # is pinned to OTLP/HTTP by the receiver below (derived in # hive-c0re/environment.nix). + vmCfg = config.services.hyperhive.swarm.victoriametrics; + # Two independent destinations, either of which may be absent: an + # upstream the operator named, and the swarm's own store when this + # host runs it. The assertion above guarantees at least one. + upstreamConfigured = otel.endpoint != ""; + localStore = vmCfg.enable; + + storeName = "otlphttp/victoriametrics"; + store = { + # ⚠️ `metrics_endpoint`, NOT `endpoint`, and the difference is + # invisible until you read the far end: `endpoint` is a BASE that + # otlphttp appends `/v1/metrics` to, while VictoriaMetrics serves + # OTLP at `/opentelemetry/api/v1/push`. With `endpoint` the + # collector still answers 200 to its own clients and the samples + # are silently posted to a path that does not exist. + # `metrics_endpoint` is used verbatim. + # + # Measured end-to-end rather than read: a real sample crossed a + # real collector into a real store, and the same probe with + # `endpoint` never arrived — see `state/probe-3265-collector-to-vm.sh`. + metrics_endpoint = "http://127.0.0.1:${toString vmCfg.port}/opentelemetry/api/v1/push"; + }; + grpcUpstream = otel.protocol == "grpc"; upstreamName = if grpcUpstream then "otlp" else "otlphttp"; upstream = { @@ -213,10 +253,17 @@ validateConfigFile = true; settings = { receivers.otlp.protocols.http.endpoint = listen; - exporters.${upstreamName} = upstream; + exporters = + lib.optionalAttrs upstreamConfigured { ${upstreamName} = upstream; } + // lib.optionalAttrs localStore { ${storeName} = store; }; service.pipelines.metrics = { receivers = [ "otlp" ]; - exporters = [ upstreamName ]; + # Fan-out, not a choice: with both configured the same + # samples go upstream AND into the swarm's store. A local + # store is for looking at this swarm; an upstream is for + # whoever aggregates across swarms, and neither replaces + # the other. + exporters = lib.optional upstreamConfigured upstreamName ++ lib.optional localStore storeName; }; }; }; diff --git a/nix/host-modules/swarm-required-services.nix b/nix/host-modules/swarm-required-services.nix index 4035ce81..480ce56e 100644 --- a/nix/host-modules/swarm-required-services.nix +++ b/nix/host-modules/swarm-required-services.nix @@ -58,5 +58,17 @@ in # mode was minting the queue's callout nkeys and then never starting # the queue they authenticate against. nats.enable = lib.mkDefault swarmCfg.enableRequiredServices; + + # The metrics pair. Once per swarm and optional, so they meet the rule + # in the option's description the same way the three above do — a hive + # that is not the service host is a *client* of this Grafana, not a + # second one. + # + # They derive together on purpose: a store with no UI is unreadable and + # a UI with no store is empty, so there is no sensible deployment that + # takes one and not the other from this switch. An operator who wants + # exactly one still sets it directly, which `mkDefault` allows. + victoriametrics.enable = lib.mkDefault swarmCfg.enableRequiredServices; + grafana.enable = lib.mkDefault swarmCfg.enableRequiredServices; }; }