feat(#3265): feed the store from the collector, and derive the pair

Review feedback: the metrics pair had nothing writing into it, and it sat
outside the switch that turns on every other swarm-wide service.

The collector now exports to VictoriaMetrics as well as upstream — a
fan-out, not a choice: a local store is for looking at this swarm, an
upstream is for whoever aggregates across swarms.

That makes a local store a complete destination on its own, so
`otel.endpoint` is no longer required when it runs here; a hive with
neither is still refused. The assertion only ever relaxes, so every
config that evaluated before still does.

`enableRequiredServices` now derives both halves, alongside matrix,
authelia and nats. They derive together because a store with no UI is
unreadable and a UI with no store is empty.

docs/swarm/services.md

@@ -68,15 +68,19 @@ one Grafana, in two containers at `metrics.<swarm-domain>` and
 `grafana.<swarm-domain>`. Two containers rather than one so Grafana can
 be restarted or broken without taking the time-series database with it.
 
-Both are **opt-in** — unlike authelia and matrix they do not follow
+Both follow `swarm.enableRequiredServices` like authelia and matrix, so
-`swarm.enableRequiredServices`, because turning them on starts a
+the swarm's service host gets them with everything else. They derive
-database that grows for as long as the swarm runs:
+together: a store with no UI is unreadable and a UI with no store is
+empty. To run one without the other, set it directly:
 
 ```nix
 services.hyperhive.swarm.victoriametrics.enable = true;
-services.hyperhive.swarm.grafana.enable = true;
+services.hyperhive.swarm.grafana.enable = false;
 ```
 
+⚠️ **This starts a database that grows for as long as the swarm runs.**
+See `retentionPeriod` below before leaving it at its default.
+
 | Option | When you'd touch it |
 |---|---|
 | `swarm.victoriametrics.retentionPeriod` | Default `5y`. Lower it once you have measured how fast this swarm actually fills a disk — the default is deliberately generous because too-short silently discards history you cannot get back. |
@@ -89,9 +93,15 @@ login form is switched off whenever SSO is configured. If you enable
 Grafana on a host with no authelia, the form stays on and Grafana's
 default `admin`/`admin` applies; change it before exposing that host.
 
-The metrics **arrive** from the swarm's OTEL collector, not from agents
+**Where the data comes from.** With `otel.enable` on, the hive's OTEL
-directly — see [`../observability.md`](../observability.md). Neither
+collector writes into this store as well as to any upstream endpoint —
-container is reachable except through the gateway: both bind loopback,
+both, not one or the other, since a local store is for looking at this
-and VictoriaMetrics' write endpoint takes no credential, so the
+swarm and an upstream is for whoever aggregates across swarms. That also
-collector is the only intended writer.
+means `otel.endpoint` is no longer required when the store runs here: a
+hive with a local store already has somewhere for telemetry to go. See
+[`../observability.md`](../observability.md).
+
+Neither container is reachable except through the gateway: both bind
+loopback, and VictoriaMetrics' write endpoint takes no credential, so
+the collector is the only intended writer.
 

nix/host-modules/otel.nix

@@ -158,8 +158,25 @@
     (lib.mkIf config.services.hyperhive.c0re.enable {
       assertions = lib.optionals config.services.hyperhive.otel.enable [
         {
-          assertion = config.services.hyperhive.otel.endpoint != "";
+          # Telemetry has to go SOMEWHERE, but "somewhere" stopped meaning
-          message = "services.hyperhive.otel.enable is true but services.hyperhive.otel.endpoint is empty.";
+          # "an upstream endpoint" once the swarm grew its own store: a hive
+          # running `swarm.victoriametrics` is a complete destination on its
+          # own, and requiring an external endpoint as well would make the
+          # all-local mode impossible to express.
+          #
+          # This only ever relaxes the old rule — every config that passed
+          # before still passes.
+          assertion =
+            config.services.hyperhive.otel.endpoint != ""
+            || config.services.hyperhive.swarm.victoriametrics.enable;
+          message = ''
+            services.hyperhive.otel.enable is true but telemetry has nowhere
+            to go: services.hyperhive.otel.endpoint is empty and
+            services.hyperhive.swarm.victoriametrics.enable is false.
+
+            Set the endpoint to export upstream, or enable the swarm's
+            metrics store to keep telemetry on this host.
+          '';
         }
       ];
     })
@@ -176,6 +193,29 @@
           # becoming "how agents talk to the collector". The agent half
           # is pinned to OTLP/HTTP by the receiver below (derived in
           # hive-c0re/environment.nix).
+          vmCfg = config.services.hyperhive.swarm.victoriametrics;
+          # Two independent destinations, either of which may be absent: an
+          # upstream the operator named, and the swarm's own store when this
+          # host runs it. The assertion above guarantees at least one.
+          upstreamConfigured = otel.endpoint != "";
+          localStore = vmCfg.enable;
+
+          storeName = "otlphttp/victoriametrics";
+          store = {
+            # ⚠️ `metrics_endpoint`, NOT `endpoint`, and the difference is
+            # invisible until you read the far end: `endpoint` is a BASE that
+            # otlphttp appends `/v1/metrics` to, while VictoriaMetrics serves
+            # OTLP at `/opentelemetry/api/v1/push`. With `endpoint` the
+            # collector still answers 200 to its own clients and the samples
+            # are silently posted to a path that does not exist.
+            # `metrics_endpoint` is used verbatim.
+            #
+            # Measured end-to-end rather than read: a real sample crossed a
+            # real collector into a real store, and the same probe with
+            # `endpoint` never arrived — see `state/probe-3265-collector-to-vm.sh`.
+            metrics_endpoint = "http://127.0.0.1:${toString vmCfg.port}/opentelemetry/api/v1/push";
+          };
+
           grpcUpstream = otel.protocol == "grpc";
           upstreamName = if grpcUpstream then "otlp" else "otlphttp";
           upstream = {
@@ -213,10 +253,17 @@
             validateConfigFile = true;
             settings = {
               receivers.otlp.protocols.http.endpoint = listen;
-              exporters.${upstreamName} = upstream;
+              exporters =
+                lib.optionalAttrs upstreamConfigured { ${upstreamName} = upstream; }
+                // lib.optionalAttrs localStore { ${storeName} = store; };
               service.pipelines.metrics = {
                 receivers = [ "otlp" ];
-                exporters = [ upstreamName ];
+                # Fan-out, not a choice: with both configured the same
+                # samples go upstream AND into the swarm's store. A local
+                # store is for looking at this swarm; an upstream is for
+                # whoever aggregates across swarms, and neither replaces
+                # the other.
+                exporters = lib.optional upstreamConfigured upstreamName ++ lib.optional localStore storeName;
               };
             };
           };

nix/host-modules/swarm-required-services.nix

@@ -58,5 +58,17 @@ in
     # mode was minting the queue's callout nkeys and then never starting
     # the queue they authenticate against.
     nats.enable = lib.mkDefault swarmCfg.enableRequiredServices;
+
+    # The metrics pair. Once per swarm and optional, so they meet the rule
+    # in the option's description the same way the three above do — a hive
+    # that is not the service host is a *client* of this Grafana, not a
+    # second one.
+    #
+    # They derive together on purpose: a store with no UI is unreadable and
+    # a UI with no store is empty, so there is no sensible deployment that
+    # takes one and not the other from this switch. An operator who wants
+    # exactly one still sets it directly, which `mkDefault` allows.
+    victoriametrics.enable = lib.mkDefault swarmCfg.enableRequiredServices;
+    grafana.enable = lib.mkDefault swarmCfg.enableRequiredServices;
   };
 }