From 5fdbf0b4523261b9ea4035a149781db6f99aca9b Mon Sep 17 00:00:00 2001
From: iris
Date: Tue, 18 Aug 2026 20:34:27 +0200
Subject: [PATCH 1/8] swarm-ui: add required hive dropdown to CreateAgentPage
Closes #3434.
Agent creation had no way to record which hive an agent runs on.
POST /api/agents now requires a hive, so the roster fetched off
GET /api/hives backs a required SelectField here rather than a
free-text field. Single-hive swarms auto-select their only hive;
multi-hive swarms show a disabled placeholder and force an explicit
choice.
Rebuilt on top of the #3448 form kit (merged after this branch was
originally opened): reuses TextField/SelectField/Button instead of
page-scoped input chrome, and SelectField gains an optional disabled
placeholder option (needed for the loading/empty/multi-hive states
here, generalizes cleanly for future callers). Form goes back to a
column layout per mara's earlier visual feedback on this same page
(two fields of different natural width no longer line up in a row).
---
.../swarm-ui/src/pages/CreateAgentPage.css | 13 ++--
.../swarm-ui/src/pages/CreateAgentPage.tsx | 70 +++++++++++++++++--
.../src/ui/select-field/SelectField.tsx | 15 ++++
3 files changed, 86 insertions(+), 12 deletions(-)
diff --git a/frontend/packages/swarm-ui/src/pages/CreateAgentPage.css b/frontend/packages/swarm-ui/src/pages/CreateAgentPage.css
index 789a6403..394ad05c 100644
--- a/frontend/packages/swarm-ui/src/pages/CreateAgentPage.css
+++ b/frontend/packages/swarm-ui/src/pages/CreateAgentPage.css
@@ -1,16 +1,19 @@
/* — the create-agent form. Input/button chrome now
- comes from the shared `ui/` form kit (`TextField`/`Button`); this
- file only owns the page's own layout + copy. Reuses the same base16
- slots (../theme.css) every other component draws from. */
+ comes from the shared `ui/` form kit (`TextField`/`SelectField`/
+ `Button`); this file only owns the page's own layout + copy. Column,
+ not row: with a second field (hive) added, a row layout put fields of
+ different natural widths on one baseline and looked misaligned —
+ mara: "make it a col". Reuses the same base16 slots (../theme.css)
+ every other component draws from. */
.create-agent-intro {
margin: 0 0 1.5em;
color: var(--muted);
}
.create-agent-form {
display: flex;
- align-items: flex-end;
+ flex-direction: column;
+ align-items: flex-start;
gap: 0.75em;
- flex-wrap: wrap;
}
.create-agent-result {
margin-top: 1em;
diff --git a/frontend/packages/swarm-ui/src/pages/CreateAgentPage.tsx b/frontend/packages/swarm-ui/src/pages/CreateAgentPage.tsx
index 35fc4722..5d6d7634 100644
--- a/frontend/packages/swarm-ui/src/pages/CreateAgentPage.tsx
+++ b/frontend/packages/swarm-ui/src/pages/CreateAgentPage.tsx
@@ -16,19 +16,35 @@
// one path segment deep — a real bug, filed separately rather than
// fixed here, but reason enough to avoid a nested route today.
//
-// First real form in this package — its name field + submit button now
-// come from the shared `ui/` form kit (`TextField`/`Button`) rather than
-// page-scoped input/button chrome, so a hive-picker soon landing on this
-// same page has something to reuse instead of copying this page's CSS.
-import { useState } from 'preact/hooks';
+// `hive` field added because agent creation had no way to record which
+// hive an agent runs on. `POST /api/agents` now requires it, so the
+// roster fetched off `GET /api/hives` backs a required select here
+// rather than a free-text field — a spawn target has to be one of the
+// swarm's actual hives, same reasoning `Ident.parse` gets client-side
+// pattern validation for `name`.
+//
+// Its name field, hive select, and submit button all come from the
+// shared `ui/` form kit (`TextField`/`SelectField`/`Button`) rather than
+// page-scoped input/button chrome — see that kit's own comments for why.
+import { useEffect, useState } from 'preact/hooks';
import { Link } from 'wouter-preact';
import { ApiErrorPanel } from '@hive/shared/api-error-panel.js';
import { readApiError, type ProblemDetails } from '@hive/shared/api-error.js';
import { Panel } from '../ui/panel/Panel.js';
import { TextField } from '../ui/text-field/TextField.js';
+import { SelectField, type SelectOption } from '../ui/select-field/SelectField.js';
import { Button } from '../ui/button/Button.js';
import './CreateAgentPage.css';
+// Mirrors swarm-controller's `HiveEntry` — same shape `OverviewPage`
+// consumes off `/api/hives/status`, but this page hits the plain
+// `/api/hives` roster (no status/freshness needed, just "what hives
+// exist to spawn into").
+interface HiveEntry {
+ name: string;
+ domain: string;
+}
+
interface CreateAgentResponse {
node_id: number;
}
@@ -56,8 +72,37 @@ const NAME_PATTERN = '[a-z0-9\\-]{1,63}';
export function CreateAgentPage() {
const [name, setName] = useState('');
+ const [hive, setHive] = useState('');
+ // `null` = still loading, `[]` = loaded but empty (a real, if unusual,
+ // swarm state) — distinct from "not fetched yet" so the placeholder
+ // option's label can tell the two apart.
+ const [hives, setHives] = useState(null);
+ const [hivesError, setHivesError] = useState(null);
const [result, setResult] = useState({ status: 'idle' });
+ // Loaded once, not re-fetched on submit: the roster changes rarely
+ // enough (a nix-level swarm config change) that staleness within one
+ // page visit isn't worth a second round-trip per keystroke/submit.
+ useEffect(() => {
+ (async () => {
+ const r = await fetch('/api/hives');
+ if (!r.ok) {
+ setHivesError(await readApiError(r));
+ return;
+ }
+ const data = (await r.json()) as HiveEntry[];
+ setHives(data);
+ // Single-hive swarms are the common case — default it rather than
+ // making the operator pick the only option. Multi-hive swarms get
+ // no default (the placeholder stays selected), so `required` on
+ // the select below forces an explicit choice per mara's scope.
+ if (data.length === 1) setHive(data[0].name);
+ })().catch((e: unknown) => setHivesError({ detail: String(e) }));
+ }, []);
+
+ const hiveOptions: SelectOption[] = (hives ?? []).map((h) => ({ value: h.name, label: h.name }));
+ const hivesReady = hives !== null && hives.length > 0;
+
async function submit(e: Event) {
e.preventDefault();
setResult({ status: 'submitting' });
@@ -65,7 +110,7 @@ export function CreateAgentPage() {
const r = await fetch('/api/agents', {
method: 'POST',
headers: { 'content-type': 'application/json' },
- body: JSON.stringify({ name }),
+ body: JSON.stringify({ name, hive }),
});
if (!r.ok) {
setResult({ status: 'error', problem: await readApiError(r) });
@@ -85,6 +130,7 @@ export function CreateAgentPage() {
Create a new agent's swarm-level identity. This only queues the job — check{' '}
jobs to watch it settle.
+ {hivesError && }
diff --git a/frontend/packages/swarm-ui/src/ui/select-field/SelectField.tsx b/frontend/packages/swarm-ui/src/ui/select-field/SelectField.tsx
index 982b0fbd..d9f66305 100644
--- a/frontend/packages/swarm-ui/src/ui/select-field/SelectField.tsx
+++ b/frontend/packages/swarm-ui/src/ui/select-field/SelectField.tsx
@@ -18,6 +18,8 @@ export function SelectField({
onChange,
options,
required,
+ disabled,
+ placeholder,
}: {
id: string;
label: string;
@@ -25,6 +27,13 @@ export function SelectField({
onChange: (value: string) => void;
options: SelectOption[];
required?: boolean;
+ disabled?: boolean;
+ // Rendered as a disabled, always-first `value=""` option — for a
+ // required select with no default (e.g. a multi-hive roster: no
+ // single right answer to pre-select), so the control shows something
+ // other than silently landing on whatever option happens to be first.
+ // Omit when the field always has a real default (`value` never `''`).
+ placeholder?: string;
}) {
return (
@@ -33,8 +42,14 @@ export function SelectField({
class="ui-form-control"
value={value}
required={required}
+ disabled={disabled}
onChange={(e) => onChange((e.target as HTMLSelectElement).value)}
>
+ {placeholder !== undefined && (
+
+ {placeholder}
+
+ )}
{options.map((o) => (
{o.label}
From f61c927310046bbc91b01bd654c585db6f1182cf Mon Sep 17 00:00:00 2001
From: atlas
Date: Tue, 18 Aug 2026 12:55:27 +0200
Subject: [PATCH 2/8] feat(#3125): a swarm-tier OTEL collector, in its own
container
The swarm tier is the only holder of the upstream credential, the only
writer to the swarm's metrics store, and (once #3283 lands) the place
that stamps hive= from the authenticated connection rather than from
anything a sender can choose. Today one collector does both tiers' jobs,
which works only because they land on one box.
A container rather than a second host unit, for two reasons that agree:
every sibling swarm service is one, and `services.opentelemetry-collector`
is a singleton NixOS option already spoken for on the host by the hive
tier. A container gets its own evaluation and therefore its own
collector.
Port defaults to 4319, deliberately not the OTLP default 4318 the hive
tier uses: swarm containers share the host netns, and two listeners
claiming one port is not a build failure but a runtime coin toss with
nothing in any log saying so -- the same collision grafana and the forge
hit on 3000.
`url` is an option with a co-located default rather than a loopback
literal in the exporter, so a split-host deployment is a config change
instead of a code change.
Wires nothing yet: the hive tier still exports directly, and switching it
over is the next commit.
---
nix/host-modules/default.nix | 1 +
nix/host-modules/swarm-otel.nix | 213 ++++++++++++++++++++++++++++++++
2 files changed, 214 insertions(+)
create mode 100644 nix/host-modules/swarm-otel.nix
diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix
index d505a6a2..666490b5 100644
--- a/nix/host-modules/default.nix
+++ b/nix/host-modules/default.nix
@@ -27,6 +27,7 @@
./swarm-nats.nix
./swarm-controller.nix
./swarm-grafana.nix
+ ./swarm-otel.nix
./swarm-snapshot-store.nix
./swarm-ui.nix
./swarm-victoriametrics.nix
diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix
new file mode 100644
index 00000000..23ae296e
--- /dev/null
+++ b/nix/host-modules/swarm-otel.nix
@@ -0,0 +1,213 @@
+# The swarm's telemetry collector: one per swarm, in a `swarm-otel`
+# nixos-container beside the swarm's other shared services.
+#
+# Two tiers, and they are separate on purpose:
+#
+# - `otel.nix` is the **hive** tier. It receives from this hive's agents
+# on the bridge and forwards, and it holds no upstream credential.
+# - this is the **swarm** tier. It is the only holder of the upstream
+# credential, the only writer to the swarm's metrics store, and (once
+# #3283 lands) the place that stamps `hive=` from the authenticated
+# connection rather than from anything a sender can choose.
+#
+# On a host that runs both, both processes run. They are not collapsed:
+# all-local is a statement about *where* processes run, not about what
+# shape the deployment has, and a local tier boundary that disappears is
+# one the local deployment stops testing. `hive=` attribution is the
+# property that would differ, and it is the one #3283 depends on.
+#
+# A container rather than a second host unit, for the same reason every
+# sibling swarm service is one — and because `services.opentelemetry-collector`
+# is a singleton NixOS option, already spoken for on the host by the hive
+# tier. A container gets its own evaluation and therefore its own collector.
+{
+ pkgs,
+ lib,
+ config,
+ ...
+}:
+let
+ cfg = config.services.hyperhive.swarm.otel;
+ swarmCfg = config.services.hyperhive.swarm;
+ otelCfg = config.services.hyperhive.otel;
+ vmCfg = config.services.hyperhive.swarm.victoriametrics;
+in
+{
+ options.services.hyperhive.swarm.otel = {
+ enable = lib.mkOption {
+ type = lib.types.bool;
+ default = swarmCfg.enableRequiredServices;
+ defaultText = lib.literalExpression "services.hyperhive.swarm.enableRequiredServices";
+ description = ''
+ Run the swarm's telemetry collector on this host.
+
+ Derived from `swarm.enableRequiredServices` like the swarm's other
+ shared services: a swarm has one of these, and it belongs wherever
+ the shared services live rather than on every hive.
+
+ A hive that does not run it still runs its own hive-tier collector
+ (`services.hyperhive.otel.enable`) and points it here with
+ {option}`services.hyperhive.swarm.otel.url`.
+ '';
+ };
+
+ machine = lib.mkOption {
+ type = lib.types.str;
+ default = "swarm-otel";
+ description = ''
+ Name of the nixos-container this collector runs in — also the
+ `machinectl` name, so other modules may read it rather than
+ repeating the literal.
+ '';
+ };
+
+ port = lib.mkOption {
+ type = lib.types.port;
+ default = 4319;
+ description = ''
+ Port this collector's OTLP/HTTP receiver listens on.
+
+ ⚠️ **Deliberately not 4318**, the OTLP/HTTP default, because the
+ hive tier already uses it (`services.hyperhive.otel.collector.port`)
+ and every swarm container shares the host's network namespace. Two
+ listeners claiming one port on one host is not a build failure —
+ it is a runtime coin toss over which one gets it, with nothing in
+ any log saying so. The same collision cost a release when grafana
+ and the forge both defaulted to 3000.
+ '';
+ };
+
+ url = lib.mkOption {
+ type = lib.types.str;
+ default = "http://127.0.0.1:${toString cfg.port}";
+ defaultText = lib.literalExpression ''"http://127.0.0.1:''${toString config.services.hyperhive.swarm.otel.port}"'';
+ description = ''
+ Where the **hive** tier sends what it receives — this collector's
+ OTLP/HTTP base URL.
+
+ The default addresses it on loopback, which is correct while the
+ two tiers share a host: every swarm container runs in the host's
+ network namespace, so a swarm service is reachable there exactly
+ as the metrics store already is.
+
+ ⚠️ That default is a *default*, not an assumption baked into the
+ exporter. A hive whose swarm collector runs elsewhere sets this to
+ that host's address, and nothing else changes — a loopback literal
+ written directly into the exporter would have made the split-host
+ case a code change instead of a config one.
+ '';
+ };
+ };
+
+ config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) {
+ assertions = [
+ {
+ # The tier exists to hold the upstream credential and to write the
+ # swarm's store. With neither, it is a process that receives
+ # samples and drops them — which looks healthy and loses data.
+ assertion = otelCfg.endpoint != "" || vmCfg.enable;
+ message = ''
+ services.hyperhive.swarm.otel.enable is true but this collector
+ has nowhere to send what it receives:
+ services.hyperhive.otel.endpoint is empty and
+ services.hyperhive.swarm.victoriametrics.enable is false.
+
+ Set the endpoint to export upstream, or enable the swarm's
+ metrics store.
+ '';
+ }
+ ];
+
+ containers.${cfg.machine} = {
+ autoStart = true;
+ ephemeral = false;
+ # Shared host netns, like every sibling swarm service: the hive tier
+ # reaches this collector, and this collector reaches the metrics
+ # store, without either crossing a network boundary that would need
+ # its own trust material.
+ privateNetwork = false;
+
+ # The upstream credential is operator-provided and lives on the host.
+ # Read-only, and only when one is configured — binding a path that
+ # does not exist makes nixos-container refuse to start the container,
+ # which is a stall several layers from its cause.
+ bindMounts = lib.optionalAttrs (otelCfg.headersCredential != null) {
+ ${otelCfg.headersCredential} = {
+ hostPath = otelCfg.headersCredential;
+ isReadOnly = true;
+ };
+ };
+
+ config =
+ { ... }:
+ {
+ system.stateVersion = config.system.stateVersion;
+ networking.firewall.enable = false;
+ # Keep the host-copied /etc/resolv.conf intact — same reasoning
+ # as the sibling swarm containers.
+ networking.resolvconf.enable = lib.mkForce false;
+
+ services.opentelemetry-collector = {
+ enable = true;
+ package = pkgs.opentelemetry-collector-contrib;
+ # Runs `otelcol validate` at build time. ⚠️ A parser, not a
+ # wiring check: it accepts a receiver naming an absent
+ # extension, and the collector then dies at startup. A green
+ # build does not prove this config starts, never mind that a
+ # sample arrives — which is why this module's gate pushes a
+ # real sample through both tiers into the store.
+ validateConfigFile = true;
+ settings = {
+ receivers.otlp.protocols.http.endpoint = "127.0.0.1:${toString cfg.port}";
+
+ exporters =
+ lib.optionalAttrs vmCfg.enable {
+ # `metrics_endpoint`, NOT `endpoint`: the latter is a
+ # base that otlphttp appends `/v1/metrics` to, while
+ # VictoriaMetrics serves OTLP at
+ # `/opentelemetry/api/v1/push`. With `endpoint` the
+ # collector answers 200 to its own clients and posts the
+ # samples to a path that does not exist. Measured
+ # end-to-end, not read — `state/probe-3265-collector-to-vm.sh`.
+ "otlphttp/victoriametrics".metrics_endpoint =
+ "http://127.0.0.1:${toString vmCfg.port}/opentelemetry/api/v1/push";
+ }
+ // lib.optionalAttrs (otelCfg.endpoint != "") {
+ ${if otelCfg.protocol == "grpc" then "otlp" else "otlphttp"} = {
+ endpoint = otelCfg.endpoint;
+ }
+ // lib.optionalAttrs (otelCfg.headersCredential != null) {
+ # Interpolated by the collector from its environment at
+ # runtime, never by nix: `EnvironmentFile` below is what
+ # puts it there, so the value is not read into the store.
+ headers.${otelCfg.collector.upstreamHeaderName} = "\${env:${otelCfg.collector.upstreamHeaderName}}";
+ }
+ // lib.optionalAttrs (otelCfg.protocol == "http/json") { encoding = "json"; };
+ };
+
+ service.pipelines.metrics = {
+ receivers = [ "otlp" ];
+ # Fan-out, not a choice: with both configured the same
+ # samples go upstream AND into the swarm's store. The store
+ # is for looking at this swarm; the upstream is for whoever
+ # aggregates across swarms, and neither replaces the other.
+ exporters =
+ lib.optional (otelCfg.endpoint != "") (if otelCfg.protocol == "grpc" then "otlp" else "otlphttp")
+ ++ lib.optional vmCfg.enable "otlphttp/victoriametrics";
+ };
+ };
+ };
+
+ # The credential file is already `NAME=value`, systemd's
+ # EnvironmentFile format — so the secret reaches the process as an
+ # environment variable without being read by nix, written to the
+ # store, or passed in argv.
+ systemd.services.opentelemetry-collector.serviceConfig =
+ lib.optionalAttrs (otelCfg.headersCredential != null)
+ {
+ EnvironmentFile = otelCfg.headersCredential;
+ };
+ };
+ };
+ };
+}
From 094a54e7857794fa9ff7c06abf2422df49b9ad47 Mon Sep 17 00:00:00 2001
From: atlas
Date: Tue, 18 Aug 2026 13:09:49 +0200
Subject: [PATCH 3/8] feat(#3125): the hive tier forwards to the swarm tier and
holds nothing
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The hive collector's only exporter becomes the swarm's collector, and
the upstream credential, the metrics-store exporter and the choice of
destination all move one tier up.
Its assertion goes with them: 'endpoint or a local store' was the right
rule while this tier picked the destination, and is the wrong one now.
A hive that runs no swarm services has neither, forwards to a swarm
collector elsewhere, and is correctly configured — the rule that
replaces it lives in swarm-otel.nix, where the destinations are.
The option descriptions here described a topology with one collector in
it: endpoint and protocol are not what agents are handed (they get the
derived first hop, see hive-c0re/environment.nix), and the credential is
not read by this tier.
---
nix/host-modules/otel.nix | 250 ++++++++++++++++----------------------
1 file changed, 102 insertions(+), 148 deletions(-)
diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix
index 7d67d5aa..6a756941 100644
--- a/nix/host-modules/otel.nix
+++ b/nix/host-modules/otel.nix
@@ -1,11 +1,18 @@
-# Hive-wide OTEL stats export. Set ONCE here at host level; the
-# meta-flake renderer (`hive-c0re/src/meta.rs::otel_config`) reads the
-# HYPERHIVE_OTEL_* env exported off hive-c0re's unit (see
-# ./hive-c0re) and injects the matching `hyperhive.otel.*` build-time
-# config into EVERY agent (mirroring the CA-cert injection), so each
-# agent's harness exports its own Claude Code stats directly to the
-# collector. There is no per-agent opt-in — this is the single switch
-# for the whole hive.
+# Hive-wide OTEL stats export, and the HIVE tier of the collector pair.
+# Set ONCE here at host level; the meta-flake renderer
+# (`hive-c0re/src/meta.rs::otel_config`) reads the HYPERHIVE_OTEL_* env
+# exported off hive-c0re's unit (see ./hive-c0re) and injects the matching
+# `hyperhive.otel.*` build-time config into EVERY agent (mirroring the
+# CA-cert injection), so each agent's harness exports its own Claude Code
+# stats directly to the collector. There is no per-agent opt-in — this is
+# the single switch for the whole hive.
+#
+# This tier receives from this hive's agents and forwards to the swarm's
+# collector (./swarm-otel.nix). It holds no credential and picks no
+# destination: an agent's samples cross a hive boundary exactly once, and
+# what happens after that is the swarm's decision, not a hive's. The
+# upstream options declared below describe that far end and are read one
+# tier up — they stay here because they mean what they have always meant.
{
lib,
config,
@@ -18,11 +25,15 @@
cost, tool calls) to an OTLP endpoint via Claude Code's built-in
OpenTelemetry. One switch for all agents.
- Enabling this also runs a collector on the host: there is exactly
- one way telemetry leaves this hive, and it is through that
+ Enabling this also runs this hive's collector on the host: there is
+ exactly one way telemetry leaves this hive, and it is through that
collector. Agents export unauthenticated to a bridge address only
- their own containers can reach, and the collector is the only
- holder of the upstream credential — an agent never sees it.
+ their own containers can reach.
+
+ That collector forwards to the swarm's
+ ({option}`services.hyperhive.swarm.otel.enable`), which holds the
+ upstream credential and writes the swarm's store. So an agent never
+ sees the credential, and neither does this tier.
⚠️ The collector is therefore in the path of all telemetry. It runs
on the same host as the agents and restarts on failure, and
@@ -36,15 +47,19 @@
default = "";
example = "https://collector.example.com/otel";
description = ''
- Upstream OTLP endpoint, set as `OTEL_EXPORTER_OTLP_ENDPOINT` for
- every agent.
+ Upstream OTLP endpoint: where telemetry ultimately goes, after it
+ has left the swarm.
- Required when `enable` is true, **unless** this host runs the
- swarm's metrics store
- ({option}`services.hyperhive.swarm.victoriametrics.enable`) — that
- store is a destination in its own right, and with both configured
- telemetry goes to both. With neither, `enable` is refused rather
- than silently exporting nowhere.
+ Read by the swarm's collector
+ ({option}`services.hyperhive.swarm.otel.enable`), which is the only
+ tier that holds the upstream credential. An agent is handed the
+ *first* hop instead — this hive's own collector — so this value is
+ never given to a container.
+
+ Optional. Leave it empty and the swarm's own metrics store
+ ({option}`services.hyperhive.swarm.victoriametrics.enable`) is the
+ destination; that is a complete deployment, not a degraded one.
+ Set both and telemetry goes to both.
'';
};
@@ -56,7 +71,12 @@
];
default = "http/protobuf";
description = ''
- OTLP wire protocol, set as `OTEL_EXPORTER_OTLP_PROTOCOL`.
+ OTLP wire protocol for the **upstream** link, honoured by the
+ swarm collector's exporter.
+
+ Not what agents speak: their first hop is this hive's collector,
+ whose OTLP/HTTP receiver takes protobuf whatever the upstream
+ wants (see `hive-c0re/environment.nix`).
'';
};
@@ -73,12 +93,12 @@
upstream auth header as `NAME=value` (e.g.
`Authorization=Bearer `).
- **Only the host-side collector reads this.** It reaches the
- collector as an `EnvironmentFile`, so the value is never read by
- nix, never copied into the store or the generated config, and
- never passed in argv — and it is never forwarded into an agent
- container, which is the point of the collector existing. Must be
- absolute.
+ **Only the swarm's collector reads this** — the one tier that
+ talks to the upstream. It arrives as an `EnvironmentFile`, so the
+ value is never read by nix, never copied into the store or the
+ generated config, and never passed in argv; and it reaches
+ neither an agent container nor this hive's own collector, which
+ is the point of the tiers existing. Must be absolute.
Leave null if the upstream needs no auth header; the collector
then sends none rather than an empty one.
@@ -161,131 +181,65 @@
};
};
- config = lib.mkMerge [
- (lib.mkIf config.services.hyperhive.c0re.enable {
- assertions = lib.optionals config.services.hyperhive.otel.enable [
- {
- # Telemetry has to go SOMEWHERE, but "somewhere" stopped meaning
- # "an upstream endpoint" once the swarm grew its own store: a hive
- # running `swarm.victoriametrics` is a complete destination on its
- # own, and requiring an external endpoint as well would make the
- # all-local mode impossible to express.
- #
- # This only ever relaxes the old rule — every config that passed
- # before still passes.
- assertion =
- config.services.hyperhive.otel.endpoint != ""
- || config.services.hyperhive.swarm.victoriametrics.enable;
- message = ''
- services.hyperhive.otel.enable is true but telemetry has nowhere
- to go: services.hyperhive.otel.endpoint is empty and
- services.hyperhive.swarm.victoriametrics.enable is false.
+ config = lib.mkIf config.services.hyperhive.otel.enable (
+ let
+ otel = config.services.hyperhive.otel;
+ listen = "${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}";
+ swarmName = "otlphttp/swarm";
+ in
+ {
+ # Reachable from agent containers and nowhere else: this opens
+ # the port on the bridge interface only.
+ services.hyperhive.network.exposeHostPorts = [ otel.collector.port ];
- Set the endpoint to export upstream, or enable the swarm's
- metrics store to keep telemetry on this host.
- '';
- }
- ];
- })
+ services.opentelemetry-collector = {
+ enable = true;
+ # `validateConfigFile` defaults to `isStorePath configFile`,
+ # and `configFile` is null on the `settings` path — so the
+ # upstream default is OFF for exactly the way this module
+ # configures it. Turning it on runs `otelcol validate` at
+ # build time, which is the collector checking its own config.
+ # ⚠️ It is a PARSER, not a wiring check, and the gap is wider
+ # than "no sample was sent": measured 2026-08-15, `validate`
+ # ACCEPTS a receiver naming an auth extension that is absent
+ # from the build, and the collector then dies at startup with
+ # `Failed to start component`. So a green build does not
+ # prove this config STARTS, never mind that a sample arrives.
+ validateConfigFile = true;
+ settings = {
+ receivers.otlp.protocols.http.endpoint = listen;
- (lib.mkIf config.services.hyperhive.otel.enable (
- (
- let
- otel = config.services.hyperhive.otel;
- listen = "${config.services.hyperhive.network.bridgeIp}:${toString otel.collector.port}";
- # `otel.protocol` describes the UPSTREAM link and always did.
- # Inserting a collector splits the path in two, and the
- # upstream half is the one that has to keep honouring it — so
- # the exporter is chosen by it, rather than the option quietly
- # becoming "how agents talk to the collector". The agent half
- # is pinned to OTLP/HTTP by the receiver below (derived in
- # hive-c0re/environment.nix).
- vmCfg = config.services.hyperhive.swarm.victoriametrics;
- # Two independent destinations, either of which may be absent: an
- # upstream the operator named, and the swarm's own store when this
- # host runs it. The assertion above guarantees at least one.
- upstreamConfigured = otel.endpoint != "";
- localStore = vmCfg.enable;
-
- storeName = "otlphttp/victoriametrics";
- store = {
- # ⚠️ `metrics_endpoint`, NOT `endpoint`, and the difference is
- # invisible until you read the far end: `endpoint` is a BASE that
- # otlphttp appends `/v1/metrics` to, while VictoriaMetrics serves
- # OTLP at `/opentelemetry/api/v1/push`. With `endpoint` the
- # collector still answers 200 to its own clients and the samples
- # are silently posted to a path that does not exist.
- # `metrics_endpoint` is used verbatim.
+ # One destination, and it is the swarm's collector. This tier
+ # holds no upstream credential and writes no store: it receives
+ # from this hive's agents and forwards, which is the whole of
+ # its job. Everything that decides where telemetry ultimately
+ # goes lives one tier up, in ./swarm-otel.nix.
+ exporters.${swarmName} = {
+ # ⚠️ Plain `endpoint`, and the exporter beside this one in
+ # ./swarm-otel.nix warns against exactly that spelling — read
+ # both before "fixing" either. The difference is the far end,
+ # not the exporter: `endpoint` is a BASE that otlphttp appends
+ # `/v1/metrics` to, which is precisely the path an OTLP/HTTP
+ # receiver serves. VictoriaMetrics is the odd one out, serving
+ # OTLP at `/opentelemetry/api/v1/push`, and that is why the
+ # store exporter needs `metrics_endpoint` while this one must
+ # not have it.
#
- # Measured end-to-end rather than read: a real sample crossed a
- # real collector into a real store, and the same probe with
- # `endpoint` never arrived — see `state/probe-3265-collector-to-vm.sh`.
- metrics_endpoint = "http://127.0.0.1:${toString vmCfg.port}/opentelemetry/api/v1/push";
+ # Addressed by the option rather than by a loopback literal:
+ # the default already points at the co-located tier, and a
+ # hive whose swarm collector lives elsewhere then names it in
+ # config instead of needing this file changed. A loopback
+ # literal is correct only while listener and caller share a
+ # netns — the assumption that cost #2860 and #3363.
+ endpoint = config.services.hyperhive.swarm.otel.url;
};
- grpcUpstream = otel.protocol == "grpc";
- upstreamName = if grpcUpstream then "otlp" else "otlphttp";
- upstream = {
- endpoint = otel.endpoint;
- }
- // lib.optionalAttrs (otel.headersCredential != null) {
- # The value is interpolated by the collector at runtime from
- # its environment, never by nix. `EnvironmentFile` below is
- # what puts it there. No credential configured means no
- # header at all — an upstream that needs no auth is a
- # legitimate deployment, and rendering `${env:…}` for a
- # variable nothing sets would send the literal.
- headers.${otel.collector.upstreamHeaderName} = "\${env:${otel.collector.upstreamHeaderName}}";
- }
- // lib.optionalAttrs (otel.protocol == "http/json") { encoding = "json"; };
- in
- {
- # Reachable from agent containers and nowhere else: this opens
- # the port on the bridge interface only.
- services.hyperhive.network.exposeHostPorts = [ otel.collector.port ];
-
- services.opentelemetry-collector = {
- enable = true;
- # `validateConfigFile` defaults to `isStorePath configFile`,
- # and `configFile` is null on the `settings` path — so the
- # upstream default is OFF for exactly the way this module
- # configures it. Turning it on runs `otelcol validate` at
- # build time, which is the collector checking its own config.
- # ⚠️ It is a PARSER, not a wiring check, and the gap is wider
- # than "no sample was sent": measured 2026-08-15, `validate`
- # ACCEPTS a receiver naming an auth extension that is absent
- # from the build, and the collector then dies at startup with
- # `Failed to start component`. So a green build does not
- # prove this config STARTS, never mind that a sample arrives.
- validateConfigFile = true;
- settings = {
- receivers.otlp.protocols.http.endpoint = listen;
- exporters =
- lib.optionalAttrs upstreamConfigured { ${upstreamName} = upstream; }
- // lib.optionalAttrs localStore { ${storeName} = store; };
- service.pipelines.metrics = {
- receivers = [ "otlp" ];
- # Fan-out, not a choice: with both configured the same
- # samples go upstream AND into the swarm's store. A local
- # store is for looking at this swarm; an upstream is for
- # whoever aggregates across swarms, and neither replaces
- # the other.
- exporters = lib.optional upstreamConfigured upstreamName ++ lib.optional localStore storeName;
- };
- };
+ service.pipelines.metrics = {
+ receivers = [ "otlp" ];
+ exporters = [ swarmName ];
};
-
- # The credential file is already `NAME=value`, which is
- # systemd's EnvironmentFile format — so the secret reaches the
- # process as an environment variable without ever being read by
- # nix, written to the store, or passed in argv.
- systemd.services.opentelemetry-collector.serviceConfig =
- lib.optionalAttrs (otel.headersCredential != null)
- {
- EnvironmentFile = otel.headersCredential;
- };
- }
- )
- ))
- ];
+ };
+ };
+ }
+ );
}
From 80c9118f879cceadcb4819681e88bcb725796919 Mon Sep 17 00:00:00 2001
From: atlas
Date: Tue, 18 Aug 2026 13:14:06 +0200
Subject: [PATCH 4/8] docs(#3125): the collector pair, and what an operator
sets on which host
observability.md described a single collector holding the upstream
credential. It also said endpoint and protocol are what agents are
handed; agents get the derived first hop, which has been true since the
collector was introduced.
The swarm tier is documented beside its sibling swarm services rather
than here, and the one line an operator must not miss - swarm.otel.url
on a hive that does not run them - is called out in both places, since
leaving it unset loses telemetry silently.
---
docs/network.md | 7 ++-
docs/observability.md | 115 ++++++++++++++++++++++++++---------------
docs/swarm/services.md | 34 +++++++++---
3 files changed, 106 insertions(+), 50 deletions(-)
diff --git a/docs/network.md b/docs/network.md
index 1e40c4b2..5ec8cd3b 100644
--- a/docs/network.md
+++ b/docs/network.md
@@ -177,8 +177,11 @@ namespace.
### Reaching host services (`exposeHostPorts`)
By default agents can only reach the host on 80/443 (+53 DNS), so a
-host-side service on another port — e.g. a dev OTEL collector for
-`services.hyperhive.otel.endpoint` (see `docs/observability.md`) — is unreachable.
+host-side service on another port — e.g. a dev OTLP collector you want
+agents to reach directly — is unreachable. (hyperhive's own telemetry
+needs none of this: `otel.enable` opens its collector's port itself, and
+`otel.endpoint` is the *upstream*, which no agent ever dials. See
+`docs/observability.md`.)
`services.hyperhive.network.exposeHostPorts = [ 4318 ];` opens each
listed TCP port `P` on the bridge-interface `allowedTCPPorts`, so an
diff --git a/docs/observability.md b/docs/observability.md
index 5e4f5b01..f2ec11d4 100644
--- a/docs/observability.md
+++ b/docs/observability.md
@@ -16,18 +16,30 @@ services.hyperhive.otel = {
};
```
-`enable` is the single gate. `endpoint` is where telemetry goes upstream —
-required when enabled *unless* this host runs the swarm's own metrics store
-(`swarm.victoriametrics.enable`), which is a destination in its own right. With
-both, telemetry goes to both. See
+`enable` is the single gate. `endpoint` is where telemetry ends up after it
+leaves the swarm — optional, because the swarm's own metrics store
+(`swarm.victoriametrics.enable`) is a destination in its own right. With both,
+telemetry goes to both. See
[`swarm/services.md`](swarm/services.md#metrics-victoriametrics--grafana).
**There is exactly one way telemetry leaves a hive: through the collector that
`enable` starts on the host.** Agents never talk to `endpoint` themselves —
they export unauthenticated to a bridge address only their own containers can
-reach, and the collector forwards upstream with the auth header. So the
-upstream credential exists in one place, on the host, and no agent ever holds
-a copy.
+reach. That collector forwards to the swarm's
+([`swarm/services.md`](swarm/services.md#telemetry-collector-otel)), which is
+the single process holding the upstream credential and the only writer to the
+swarm's store. No agent holds a copy, and neither does this hive.
+
+⚠️ **On a hive that does not run the swarm's services, say where that swarm
+collector is:**
+
+```nix
+services.hyperhive.swarm.otel.url = "http://services-host.example:4319";
+```
+
+Left unset it points at this host, where nothing is listening — the collector
+starts, agents export happily, and the samples go nowhere. The service host
+itself needs no such line.
⚠️ **The collector is therefore in the path of all telemetry.** It runs on the
same host as the agents and restarts on failure, and telemetry is not the
@@ -42,8 +54,8 @@ port on the bridge interface only. So "unauthenticated to a bridge address"
means *reachable from an agent container*, not *presents a credential*.
The consequence, stated because it is a choice rather than an oversight: **any
-agent can push arbitrary OTLP, and the collector forwards it upstream under the
-operator's credential.** It cannot tell a container's genuine Claude Code stats
+agent can push arbitrary OTLP, and it is forwarded on under the operator's
+credential.** Neither tier can tell a container's genuine Claude Code stats
from anything else shaped like OTLP arriving on that port — including data
smuggled out in resource attributes on an otherwise-legitimate export.
@@ -61,33 +73,44 @@ closed by this design, and nothing here should be read as closing it.
Master switch. When true, all other options below take effect.
-### `services.hyperhive.otel.endpoint` — string, required when enabled unless the swarm store runs here
+### `services.hyperhive.otel.endpoint` — string, default `""`
-Upstream OTLP endpoint URL. Set as `OTEL_EXPORTER_OTLP_ENDPOINT` for every
-agent. Example: `"https://collector.example.com/otel"`.
+Upstream OTLP endpoint URL, read by the swarm's collector. Example:
+`"https://collector.example.com/otel"`.
-Leave it empty **only** on a host running `swarm.victoriametrics.enable` — the
-local store is then the destination and the collector writes there instead.
-With neither, `enable` is refused at eval: telemetry with nowhere to go is a
-misconfiguration, not a quiet no-op.
+Leave it empty on a swarm running its own metrics store — that store is then
+the destination. With neither, the swarm collector is refused at eval:
+telemetry with nowhere to go is a misconfiguration, not a quiet no-op.
+
+Not what agents are handed. Their endpoint is this hive's own collector,
+derived from the bridge address, so setting this changes where telemetry
+*ends up* and never what a container is told.
### `services.hyperhive.otel.protocol` — enum, default `"http/protobuf"`
-OTLP wire protocol, passed as `OTEL_EXPORTER_OTLP_PROTOCOL`. Accepted values:
+Wire protocol for the **upstream** link, honoured by the swarm collector's
+exporter. Accepted values:
- `"http/protobuf"` (default)
- `"http/json"`
- `"grpc"`
+Agents are not affected: their first hop is this hive's collector, whose
+OTLP/HTTP receiver takes protobuf whatever the upstream wants.
+
### `services.hyperhive.otel.headersCredential` — string or null, default `null`
Absolute path to a secret file on the host holding the upstream auth header as
`NAME=value` (e.g. `Authorization=Bearer `).
-**Only the host collector reads it.** It arrives as an `EnvironmentFile` on the
-collector's unit, so the value is never read by nix, never copied into the
-store or the generated config, never passed in argv — and **never forwarded
-into an agent container**. An agent cannot read the hive's upstream credential
-because it is never given one.
+**Only the swarm's collector reads it** — the one tier that talks to the
+upstream. It arrives as an `EnvironmentFile` on that unit, so the value is
+never read by nix, never copied into the store or the generated config, never
+passed in argv — and reaches **neither an agent container nor a hive's own
+collector**. An agent cannot read the upstream credential because it is never
+given one.
+
+Set it on the host running the swarm's services; a hive that only forwards has
+no use for it.
Leave `null` if the upstream needs no auth header; the collector then sends
none rather than an empty one.
@@ -130,46 +153,56 @@ metrics on process exit, so interval tuning is not required for metrics to be
exported. A lower value gives more frequent intermediate flushes within
long-running turns — cosmetic, not a correctness knob.
-## The host collector
+## The two collectors
-`enable` starts an OpenTelemetry collector on the host. It is not optional and
-there is no second path — that is the whole point:
+Telemetry crosses two collectors, and which one you configure depends on what
+the host is:
+
+| | runs where | receives from | does |
+|---|---|---|---|
+| **hive tier** — `otel.enable` | every hive with agents | that hive's agents, on the bridge | forwards to the swarm tier. Holds no credential, picks no destination |
+| **swarm tier** — `swarm.otel.enable` | once per swarm | every hive's collector | writes the swarm's store and exports upstream |
+
+An all-local host runs both, and needs nothing said about the hop between them.
```nix
services.hyperhive.otel = {
enable = true;
endpoint = "https://collector.example.com/otel"; # the upstream
- headersCredential = "/run/secrets/otel-headers"; # only the host reads it
+ headersCredential = "/run/secrets/otel-headers"; # only the swarm tier reads it
};
```
-**Why it isn't a knob.** Exporting straight to `endpoint` means every agent
-needs the credential to authenticate — and the harness delivers that token into
-the agent's own `~/.claude/settings.json`, a file the agent can read. `0600`
-protects it from other containers, not from the agent itself. As long as the
-direct path stays *selectable*, that hole stays selectable; an option that can
-reintroduce it is a hole with extra steps.
+**Why the hive tier isn't optional.** Exporting straight to `endpoint` means
+every agent needs the credential to authenticate — and the harness delivers
+that token into the agent's own `~/.claude/settings.json`, a file the agent can
+read. `0600` protects it from other containers, not from the agent itself. As
+long as the direct path stays *selectable*, that hole stays selectable; an
+option that can reintroduce it is a hole with extra steps.
-**`endpoint` keeps meaning "where telemetry goes upstream."** The collector
-does not redefine it — the agent-facing value is *derived*
+**Why the tiers stay separate on one box.** They are not collapsed when
+co-located: an all-local hive is a statement about *where* processes run, not
+about the shape of the deployment. A boundary that disappears locally is one
+the local deployment stops testing.
+
+**`endpoint` keeps meaning "where telemetry goes upstream."** Neither tier
+redefines it — the agent-facing value is *derived*
(`http://:`), so an existing deployment's `endpoint`
keeps working unchanged. The bridge port is contributed to `exposeHostPorts`
automatically; there is nothing to open by hand.
-What the collector *added* is a second destination: on a host running the
-swarm's metrics store it writes there too, so `endpoint` is no longer the only
-place telemetry can land — and no longer the only way to have one.
-
### `services.hyperhive.otel.collector.port` — port, default `4318`
-The OTLP/HTTP port the collector listens on, bound to the bridge IP only.
+The OTLP/HTTP port the hive tier listens on, bound to the bridge IP only. The
+swarm tier has its own (`swarm.otel.port`, default `4319`) — they share a
+network namespace when co-located, so the two must differ.
### `services.hyperhive.otel.collector.upstreamHeaderName` — string, default `"Authorization"`
-Name of the header the collector sends upstream. The **value** comes from the
+Name of the header the swarm tier sends upstream. The **value** comes from the
credential file at runtime (`EnvironmentFile` → `${env:}`), never from
nix — so header names are config and header values are secrets, which is the
-only split the collector's static header map can express.
+only split a static header map can express.
⚠️ **`endpoint` must be valid for `protocol`.** The upstream exporter follows
`otel.protocol` (`grpc` → the gRPC exporter, otherwise OTLP/HTTP), and the gRPC
diff --git a/docs/swarm/services.md b/docs/swarm/services.md
index 8608cb6c..23742b27 100644
--- a/docs/swarm/services.md
+++ b/docs/swarm/services.md
@@ -93,15 +93,35 @@ login form is switched off whenever SSO is configured. If you enable
Grafana on a host with no authelia, the form stays on and Grafana's
default `admin`/`admin` applies; change it before exposing that host.
-**Where the data comes from.** With `otel.enable` on, the hive's OTEL
-collector writes into this store as well as to any upstream endpoint —
-both, not one or the other, since a local store is for looking at this
-swarm and an upstream is for whoever aggregates across swarms. That also
-means `otel.endpoint` is no longer required when the store runs here: a
-hive with a local store already has somewhere for telemetry to go. See
-[`../observability.md`](../observability.md).
+**Where the data comes from.** The swarm's OTEL collector, below.
Neither container is reachable except through the gateway: both bind
loopback, and VictoriaMetrics' write endpoint takes no credential, so
the collector is the only intended writer.
+### Telemetry collector (OTEL)
+
+The swarm's collector receives from every hive's own collector and is the
+only process that decides where telemetry goes: it writes the store above
+and exports to `otel.endpoint`, doing both when both are configured. It
+also holds the upstream credential, which is why no hive and no agent
+needs one.
+
+It follows `swarm.enableRequiredServices` like the services above, in a
+`swarm-otel` container. Its `swarm.otel.port` defaults to `4319` rather
+than OTLP's usual `4318`, which the hive tier already uses — swarm
+containers share the host's network namespace, so two collectors on one
+port is a coin toss at runtime rather than an error at build time.
+
+| Option | When you'd touch it |
+|---|---|
+| `swarm.otel.url` | **On every hive that does not run the swarm's services.** It defaults to this host, so a hive left at the default forwards into nothing and loses its telemetry silently. Point it at the services host: `"http://services-host.example:4319"`. |
+| `swarm.otel.port` | Only if something else on the services host already claims `4319`. |
+
+With neither `otel.endpoint` nor the store enabled, this collector is
+refused at eval — a tier that receives samples and drops them looks
+healthy while losing data.
+
+Agent-side configuration, and what a hive's own collector does, are in
+[`../observability.md`](../observability.md).
+
From 636dba8d618644fd7105946eacf11d33d6f055fe Mon Sep 17 00:00:00 2001
From: atlas
Date: Tue, 18 Aug 2026 13:24:59 +0200
Subject: [PATCH 5/8] fix(#3125): the swarm tier's self-metrics must not fight
the hive tier's
A collector serves its own metrics on localhost:8888 unless told
otherwise, and co-located tiers share a network namespace, so the second
one to start dies with 'bind: address already in use'.
The port appears in neither config - it is a default inside the binary -
so comparing the ports the configs name reports them distinct. A
behavioural probe found it by being unable to start the chain.
metrics.address is the spelling that looks right and is rejected by this
version ('migration.MetricsConfigV030' has invalid keys: address);
readers is the schema it accepts.
---
nix/host-modules/swarm-otel.nix | 37 +++++++++++++++++++++++++++++++++
1 file changed, 37 insertions(+)
diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix
index 23ae296e..22ebe688 100644
--- a/nix/host-modules/swarm-otel.nix
+++ b/nix/host-modules/swarm-otel.nix
@@ -77,6 +77,26 @@ in
'';
};
+ telemetryPort = lib.mkOption {
+ type = lib.types.port;
+ default = 8889;
+ description = ''
+ Port this collector serves its **own** metrics on — queue depth,
+ refused and dropped samples, exporter failures. How you find out
+ that telemetry is being lost, so it is worth keeping rather than
+ switching off.
+
+ ⚠️ **Deliberately not 8889's neighbour 8888**, which is the
+ collector's built-in default and therefore what the hive tier
+ already binds. Two collectors share a network namespace whenever
+ they are co-located, and unlike the OTLP port this one appears
+ nowhere in either config — it is a default inside the binary, so
+ nothing that compares configured ports can see the clash. The
+ second collector to start simply dies with
+ `bind: address already in use`.
+ '';
+ };
+
url = lib.mkOption {
type = lib.types.str;
default = "http://127.0.0.1:${toString cfg.port}";
@@ -185,6 +205,23 @@ in
// lib.optionalAttrs (otelCfg.protocol == "http/json") { encoding = "json"; };
};
+ # Moves this collector's self-metrics off the built-in
+ # default of `localhost:8888`, which the hive tier holds.
+ #
+ # ⚠️ `metrics.address` is the spelling that looks right and
+ # is REJECTED by this collector version — measured, not
+ # read: `'migration.MetricsConfigV030' has invalid keys:
+ # address`. `readers` is the schema it accepts, and the
+ # difference is a startup failure rather than a warning.
+ service.telemetry.metrics.readers = [
+ {
+ pull.exporter.prometheus = {
+ host = "127.0.0.1";
+ port = cfg.telemetryPort;
+ };
+ }
+ ];
+
service.pipelines.metrics = {
receivers = [ "otlp" ];
# Fan-out, not a choice: with both configured the same
From 98ab0ad59fe9af271c1813e6306604323fa7958b Mon Sep 17 00:00:00 2001
From: atlas
Date: Tue, 18 Aug 2026 13:29:11 +0200
Subject: [PATCH 6/8] style: drop tracker tags from the collector modules'
comments
The pre-push lint refuses them, and rightly: a comment that names an
issue number ages into a pointer at a closed thread. The constraint each
one carried is stated directly instead.
---
nix/host-modules/otel.nix | 3 ++-
nix/host-modules/swarm-otel.nix | 9 +++++----
2 files changed, 7 insertions(+), 5 deletions(-)
diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix
index 6a756941..11ca025d 100644
--- a/nix/host-modules/otel.nix
+++ b/nix/host-modules/otel.nix
@@ -230,7 +230,8 @@
# hive whose swarm collector lives elsewhere then names it in
# config instead of needing this file changed. A loopback
# literal is correct only while listener and caller share a
- # netns — the assumption that cost #2860 and #3363.
+ # netns, an assumption that has cost this project two
+ # outages.
endpoint = config.services.hyperhive.swarm.otel.url;
};
diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix
index 22ebe688..40331543 100644
--- a/nix/host-modules/swarm-otel.nix
+++ b/nix/host-modules/swarm-otel.nix
@@ -6,15 +6,16 @@
# - `otel.nix` is the **hive** tier. It receives from this hive's agents
# on the bridge and forwards, and it holds no upstream credential.
# - this is the **swarm** tier. It is the only holder of the upstream
-# credential, the only writer to the swarm's metrics store, and (once
-# #3283 lands) the place that stamps `hive=` from the authenticated
-# connection rather than from anything a sender can choose.
+# credential, the only writer to the swarm's metrics store, and the
+# place that will stamp `hive=` from the authenticated connection
+# rather than from anything a sender can choose.
#
# On a host that runs both, both processes run. They are not collapsed:
# all-local is a statement about *where* processes run, not about what
# shape the deployment has, and a local tier boundary that disappears is
# one the local deployment stops testing. `hive=` attribution is the
-# property that would differ, and it is the one #3283 depends on.
+# property that would differ, and the ingest auth that makes it
+# unforgeable is built on this boundary existing.
#
# A container rather than a second host unit, for the same reason every
# sibling swarm service is one — and because `services.opentelemetry-collector`
From 28623e5effcfcfcfddd324a6e499b4f1222eab70 Mon Sep 17 00:00:00 2001
From: atlas
Date: Tue, 18 Aug 2026 13:35:05 +0200
Subject: [PATCH 7/8] fix(#3125): the swarm collector writes its own resolver,
like its siblings
All four sibling swarm containers import swarm-container-resolver.nix;
this one did not. It matters more here than most: otel.endpoint is an
operator-configured external hostname, and reaching it is the entire
reason this container holds a credential.
Also aligns two details with those siblings - the enable default is
asserted from swarm-required-services.nix with the metrics pair it
feeds, so that file remains the one place a service host is declared,
and machine is readOnly since its description already calls it a fact
rather than a knob.
---
nix/host-modules/swarm-otel.nix | 24 ++++++++++++++++----
nix/host-modules/swarm-required-services.nix | 6 +++++
2 files changed, 25 insertions(+), 5 deletions(-)
diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix
index 40331543..7460c85e 100644
--- a/nix/host-modules/swarm-otel.nix
+++ b/nix/host-modules/swarm-otel.nix
@@ -37,14 +37,14 @@ in
options.services.hyperhive.swarm.otel = {
enable = lib.mkOption {
type = lib.types.bool;
- default = swarmCfg.enableRequiredServices;
- defaultText = lib.literalExpression "services.hyperhive.swarm.enableRequiredServices";
+ default = false;
description = ''
Run the swarm's telemetry collector on this host.
- Derived from `swarm.enableRequiredServices` like the swarm's other
- shared services: a swarm has one of these, and it belongs wherever
- the shared services live rather than on every hive.
+ Asserted from `swarm.enableRequiredServices` in
+ ./swarm-required-services.nix, with the metrics pair this
+ collector feeds: a swarm has one of these, and it belongs
+ wherever the shared services live rather than on every hive.
A hive that does not run it still runs its own hive-tier collector
(`services.hyperhive.otel.enable`) and points it here with
@@ -54,6 +54,7 @@ in
machine = lib.mkOption {
type = lib.types.str;
+ readOnly = true;
default = "swarm-otel";
description = ''
Name of the nixos-container this collector runs in — also the
@@ -162,6 +163,19 @@ in
config =
{ ... }:
{
+ # This tier is the one that resolves an operator-configured
+ # hostname: `otel.endpoint` is an external URL, and reaching it
+ # is the entire reason this container holds a credential. The
+ # `/etc/resolv.conf` nixos-containers copies in is a snapshot
+ # taken once at boot, so without this the upstream export
+ # depends on the host's file having been right at that instant.
+ imports = [
+ (import ./swarm-container-resolver.nix {
+ inherit (config.services.hyperhive.network) bridgeIp;
+ dnsConsumers = [ "opentelemetry-collector.service" ];
+ })
+ ];
+
system.stateVersion = config.system.stateVersion;
networking.firewall.enable = false;
# Keep the host-copied /etc/resolv.conf intact — same reasoning
diff --git a/nix/host-modules/swarm-required-services.nix b/nix/host-modules/swarm-required-services.nix
index 81b2c779..f12d84dd 100644
--- a/nix/host-modules/swarm-required-services.nix
+++ b/nix/host-modules/swarm-required-services.nix
@@ -70,6 +70,12 @@ in
# exactly one still sets it directly, which `mkDefault` allows.
victoriametrics.enable = lib.mkDefault swarmCfg.enableRequiredServices;
grafana.enable = lib.mkDefault swarmCfg.enableRequiredServices;
+
+ # The collector that feeds the pair above, and the only tier holding
+ # the upstream credential. Same rule as the rest: once per swarm,
+ # optional, and a hive that is not the service host is a *client* of
+ # it (`swarm.otel.url`) rather than a second one.
+ otel.enable = lib.mkDefault swarmCfg.enableRequiredServices;
};
# The collector that feeds the pair above (note: no `swarm.` prefix,
From d2fb4bff7914901674085795deab352b355d8e79 Mon Sep 17 00:00:00 2001
From: atlas
Date: Tue, 18 Aug 2026 20:45:13 +0200
Subject: [PATCH 8/8] feat(#3125): reshape the hive-to-swarm OTEL hop by domain
Drops swarm.otel.url (a loopback default an operator had to override on a
split host) in favor of swarm.otel.domain -- the same
gateway.localNames + nginx-vhost-through-the-gateway shape every other
swarm service (authelia, grafana, victoriametrics, ui) already uses. The
hive tier's exporter now reaches it as https:// unconditionally,
resolved locally by dnsmasq on a co-located host and over the real network
otherwise, instead of a config knob nobody sets until they hit the silent
drop.
Costs CA trust on the hive tier: otel.nix wires
lib/hive-ca-trust.nix's trustBundle with hostUnit = true on the
opentelemetry-collector host unit, the same flag #3441/#3442 added for
swarm-controller and hive-c0re.
mara, #3125 comment 58363: "go c".
---
docs/observability.md | 15 ++---
docs/swarm/services.md | 9 ++-
nix/host-modules/otel.nix | 47 ++++++++++++---
nix/host-modules/swarm-otel.nix | 60 ++++++++++++++------
nix/host-modules/swarm-required-services.nix | 2 +-
5 files changed, 96 insertions(+), 37 deletions(-)
diff --git a/docs/observability.md b/docs/observability.md
index f2ec11d4..19daedc6 100644
--- a/docs/observability.md
+++ b/docs/observability.md
@@ -30,16 +30,11 @@ reach. That collector forwards to the swarm's
the single process holding the upstream credential and the only writer to the
swarm's store. No agent holds a copy, and neither does this hive.
-⚠️ **On a hive that does not run the swarm's services, say where that swarm
-collector is:**
-
-```nix
-services.hyperhive.swarm.otel.url = "http://services-host.example:4319";
-```
-
-Left unset it points at this host, where nothing is listening — the collector
-starts, agents export happily, and the samples go nowhere. The service host
-itself needs no such line.
+The hive collector reaches the swarm collector by its gateway name
+(`swarm.otel.domain`, default `otel.`) — the same DNS-and-CA-trust
+shape every hive-to-swarm-service hop uses, not a URL an operator has to point
+anywhere. A hive that does not run the swarm's services still resolves that
+name through the gateway; nothing here needs setting for the split-host case.
⚠️ **The collector is therefore in the path of all telemetry.** It runs on the
same host as the agents and restarts on failure, and telemetry is not the
diff --git a/docs/swarm/services.md b/docs/swarm/services.md
index 23742b27..bb5dc93e 100644
--- a/docs/swarm/services.md
+++ b/docs/swarm/services.md
@@ -113,9 +113,16 @@ than OTLP's usual `4318`, which the hive tier already uses — swarm
containers share the host's network namespace, so two collectors on one
port is a coin toss at runtime rather than an error at build time.
+Every hive's own collector reaches this one by its gateway name,
+`swarm.otel.domain` (default `otel.`) — the same
+by-domain-through-the-gateway shape every other swarm service uses, not a
+loopback URL an operator has to redirect. There is nothing to set on a hive
+that does not run the swarm's services; the name resolves through the
+gateway either way.
+
| Option | When you'd touch it |
|---|---|
-| `swarm.otel.url` | **On every hive that does not run the swarm's services.** It defaults to this host, so a hive left at the default forwards into nothing and loses its telemetry silently. Point it at the services host: `"http://services-host.example:4319"`. |
+| `swarm.otel.domain` | Only to rename it — the default already resolves correctly for every hive in the swarm. |
| `swarm.otel.port` | Only if something else on the services host already claims `4319`. |
With neither `otel.endpoint` nor the store enabled, this collector is
diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix
index 11ca025d..53d43329 100644
--- a/nix/host-modules/otel.nix
+++ b/nix/host-modules/otel.nix
@@ -14,11 +14,41 @@
# upstream options declared below describe that far end and are read one
# tier up — they stay here because they mean what they have always meant.
{
+ pkgs,
lib,
config,
...
}:
+let
+ # This tier now reaches the swarm's collector by name through the
+ # gateway (`swarm-otel.nix`'s `domain`) instead of a loopback URL, so it
+ # needs the same hive-CA trust every other host consumer of an `https://`
+ # swarm-service name needs — see `swarm-controller.nix` for the sibling
+ # wiring this copies.
+ #
+ # `hostUnit`: `opentelemetry-collector` is a host systemd service, not a
+ # container, so it reads the CA from the host path and the bundle oneshot
+ # waits on `hive-tls-ca.service` itself. `enable`: `imports` is
+ # unconditional at the host's top level, so without it a hive with this
+ # tier off would still get a bundle oneshot and a phantom
+ # `opentelemetry-collector` service holding an `SSL_CERT_FILE`.
+ caTrust = import ./lib/hive-ca-trust.nix {
+ inherit lib;
+ tlsCfg = config.services.hyperhive.tls;
+ gatewayCfg = config.services.hyperhive.gateway;
+ };
+in
{
+ imports = [
+ (caTrust.trustBundle {
+ inherit pkgs;
+ name = "hive-otel";
+ consumers = [ "opentelemetry-collector" ];
+ hostUnit = true;
+ enable = config.services.hyperhive.otel.enable;
+ })
+ ];
+
options.services.hyperhive.otel = {
enable = lib.mkEnableOption ''
hive-wide export of every agent's Claude Code stats (token usage,
@@ -225,14 +255,15 @@
# store exporter needs `metrics_endpoint` while this one must
# not have it.
#
- # Addressed by the option rather than by a loopback literal:
- # the default already points at the co-located tier, and a
- # hive whose swarm collector lives elsewhere then names it in
- # config instead of needing this file changed. A loopback
- # literal is correct only while listener and caller share a
- # netns, an assumption that has cost this project two
- # outages.
- endpoint = config.services.hyperhive.swarm.otel.url;
+ # By name through the gateway, not a loopback literal: a
+ # loopback literal is correct only while listener and caller
+ # share a netns, an assumption that has cost this project two
+ # outages, and it is exactly the split-host case a swarm
+ # service name exists to make a config fact rather than a code
+ # change. `https://` because that name resolves through the
+ # gateway even on a co-located host — see `caTrust` above for
+ # the trust half that makes this verify.
+ endpoint = "https://${config.services.hyperhive.swarm.otel.domain}";
};
service.pipelines.metrics = {
diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix
index 7460c85e..0eacc0de 100644
--- a/nix/host-modules/swarm-otel.nix
+++ b/nix/host-modules/swarm-otel.nix
@@ -32,6 +32,14 @@ let
swarmCfg = config.services.hyperhive.swarm;
otelCfg = config.services.hyperhive.otel;
vmCfg = config.services.hyperhive.swarm.victoriametrics;
+ hyperhiveCfg = config.services.hyperhive;
+ gatewayCfg = hyperhiveCfg.gateway;
+ swarmDomain = hyperhiveCfg.swarm.domain;
+
+ # Total on a null swarm domain for the same reason every sibling module is:
+ # the required-domain assertion in hive-network.nix should be what an
+ # operator sees, not a coercion error from here.
+ domainBase = if swarmDomain == null then "invalid" else swarmDomain;
in
{
options.services.hyperhive.swarm.otel = {
@@ -47,8 +55,8 @@ in
wherever the shared services live rather than on every hive.
A hive that does not run it still runs its own hive-tier collector
- (`services.hyperhive.otel.enable`) and points it here with
- {option}`services.hyperhive.swarm.otel.url`.
+ (`services.hyperhive.otel.enable`) and reaches this one by name, at
+ {option}`services.hyperhive.swarm.otel.domain`.
'';
};
@@ -99,29 +107,47 @@ in
'';
};
- url = lib.mkOption {
+ domain = lib.mkOption {
type = lib.types.str;
- default = "http://127.0.0.1:${toString cfg.port}";
- defaultText = lib.literalExpression ''"http://127.0.0.1:''${toString config.services.hyperhive.swarm.otel.port}"'';
+ default = "otel.${domainBase}";
+ defaultText = lib.literalExpression ''"otel.''${services.hyperhive.swarm.domain}"'';
description = ''
- Where the **hive** tier sends what it receives — this collector's
- OTLP/HTTP base URL.
+ Name the gateway serves this on. A sibling of the swarm's other
+ service names, so the swarm-services sub-CA can issue for it — see
+ `hive-tls.nix` for why a service name being a sibling rather than a
+ child decides which CA may sign it.
- The default addresses it on loopback, which is correct while the
- two tiers share a host: every swarm container runs in the host's
- network namespace, so a swarm service is reachable there exactly
- as the metrics store already is.
-
- ⚠️ That default is a *default*, not an assumption baked into the
- exporter. A hive whose swarm collector runs elsewhere sets this to
- that host's address, and nothing else changes — a loopback literal
- written directly into the exporter would have made the split-host
- case a code change instead of a config one.
+ This is what the **hive** tier's exporter reaches — the hive
+ collector is a plain producer against this name exactly like every
+ other client of a swarm service, resolved locally by dnsmasq on a
+ co-located host and over the real network otherwise. There is no
+ separate loopback-vs-remote knob to get wrong: `swarm-nats` is the
+ deliberate exception to this pattern (its cross-hive reach is the
+ wireguard mesh, not the gateway), everything else in this swarm
+ addresses its siblings by name.
'';
};
};
config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) {
+ # The gateway name, inside `cfg.enable` — that guard is the load-bearing
+ # part. Every hive in a swarm may know this collector exists, but only
+ # the host that RUNS it may claim the name; a client hive declaring the
+ # vhost would answer for a service it does not have.
+ services.hyperhive.gateway.localNames = [ cfg.domain ];
+
+ # OTLP/HTTP, not a browsable UI, but the same reverse-proxy shape as
+ # every sibling swarm service: TLS terminates here, then plain http to
+ # the co-located container over loopback (shared netns, like the store
+ # this collector writes to).
+ services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // {
+ listen = gatewayCfg.lib.listen;
+ extraConfig = gatewayCfg.lib.securityHeaders;
+ locations."/" = {
+ proxyPass = "http://127.0.0.1:${toString cfg.port}";
+ };
+ };
+
assertions = [
{
# The tier exists to hold the upstream credential and to write the
diff --git a/nix/host-modules/swarm-required-services.nix b/nix/host-modules/swarm-required-services.nix
index f12d84dd..80c3800a 100644
--- a/nix/host-modules/swarm-required-services.nix
+++ b/nix/host-modules/swarm-required-services.nix
@@ -74,7 +74,7 @@ in
# The collector that feeds the pair above, and the only tier holding
# the upstream credential. Same rule as the rest: once per swarm,
# optional, and a hive that is not the service host is a *client* of
- # it (`swarm.otel.url`) rather than a second one.
+ # it (by name, `swarm.otel.domain`) rather than a second one.
otel.enable = lib.mkDefault swarmCfg.enableRequiredServices;
};