diff --git a/docs/networking/network.md b/docs/networking/network.md index ebd5f4cf..658f8f7a 100644 --- a/docs/networking/network.md +++ b/docs/networking/network.md @@ -183,7 +183,7 @@ host-side service on another port — for example a dev OTLP collector you want agents to reach directly — is unreachable. (hyperhive's own services need none of this: a module that means to be reachable from agents opens its own port here, which the hive collector and `nats.enable` both do. Note -that `otel.endpoint` is the _upstream_, which no agent ever dials. See +that `deploy.swarm-otel.upstream.endpoint` is the _upstream_, which no agent ever dials. See `docs/scheduler/observability.md`.) `services.hyperhive.network.exposeHostPorts = [ 4318 ];` opens each diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index ab701adf..66212053 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -23,7 +23,7 @@ the host is: An all-local host runs both; the hop between them configures itself. ```nix -services.hyperhive.otel = { +services.hyperhive.deploy.swarm-otel.upstream = { endpoint = "https://collector.example.com/otel"; # the upstream headersCredential = "/run/secrets/otel-headers"; # only the swarm tier reads it }; @@ -33,13 +33,13 @@ services.hyperhive.otel = { Every hive runs its collector and every agent exports to it, with nothing to switch on: there is no per-hive or per-agent opt-in. -`endpoint` is where telemetry ends up after it leaves the swarm — optional, +`upstream.endpoint` is where telemetry ends up after it leaves the swarm — optional, because the swarm's own metrics store (`deploy.victoriametrics`) is a destination in its own right. With both, telemetry goes to both. See [`swarm/services.md`](../swarm/services.md#metrics-victoriametrics--grafana). **Telemetry leaves a hive exactly one way: through the collector that -runs on the host.** Agents never talk to `endpoint` themselves — +runs on the host.** Agents never talk to `upstream.endpoint` themselves — they export unauthenticated to a collector address only their own containers can reach. They dial it as `http://hive-collector.hive:4318`, a name identical on @@ -62,7 +62,7 @@ the control plane, so degraded telemetry isn't degraded operation. ### Why two tiers -**Agents export only to the hive tier.** Exporting straight to `endpoint` would mean +**Agents export only to the hive tier.** Exporting straight to `upstream.endpoint` would mean every agent needs the credential — and the only place to hand it to an agent container is somewhere the agent itself can read, its own claude settings among them. `0600` protects a secret from other containers, not from the @@ -159,8 +159,8 @@ belongs to exactly one hive. Getting the secret wrong shows up as the hive's collector logging 401s from the swarm tier and no metrics appearing for that hive. -⚠️ **`endpoint` must be valid for `protocol`.** The upstream exporter follows -`otel.protocol` (`grpc` → the gRPC exporter, otherwise OTLP/HTTP), and the gRPC +⚠️ **`upstream.endpoint` must be valid for `upstream.protocol`.** The upstream exporter follows +`upstream.protocol` (`grpc` → the gRPC exporter, otherwise OTLP/HTTP), and the gRPC exporter takes an _address_: `https://host/path` is a legal `OTEL_EXPORTER_OTLP_ENDPOINT` for HTTP but fails as gRPC with _"missing port in address"_. nix validates the collector's config at build time, so a mismatch is @@ -175,7 +175,8 @@ To let agents reach some other host-local service of your own → ## Options reference -The nix module (`nix/host-modules/otel.nix`) generates every +The nix modules (`nix/host-modules/otel.nix`, and `nix/host-modules/swarm-otel.nix` +for the `services.hyperhive.deploy.swarm-otel.upstream.*` export) generate every `services.hyperhive.otel.*` option's full type, default, description and example into the [options reference](https://hyperhive.darkest.space/options/) (host options — `nix build .#docs-host` for a local render). The same module diff --git a/docs/swarm/services.md b/docs/swarm/services.md index dd650ea9..d93a87ee 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -313,8 +313,8 @@ read side stays open to any authenticated caller. The swarm's collector receives from every hive's own collector and is the only process that decides where telemetry goes: it always writes the stores -above, and exports to `otel.endpoint` as well when that's set. Which signals -go there is `otel.upstreamSignals` (default `[ "metrics" "logs" ]`); the stores +above, and exports to `deploy.swarm-otel.upstream.endpoint` as well when that's set. Which signals +go there is `deploy.swarm-otel.upstream.signals` (default `[ "metrics" "logs" ]`); the stores always get both. It holds the upstream credential, which is why no hive and no agent needs one. diff --git a/nix/host-modules/hive-c0re/default.nix b/nix/host-modules/hive-c0re/default.nix index 557b497b..c2603a70 100644 --- a/nix/host-modules/hive-c0re/default.nix +++ b/nix/host-modules/hive-c0re/default.nix @@ -145,7 +145,8 @@ in # # ⚠️ `SSL_CERT_FILE` REPLACES the trust store, and unlike the other # consumers of this helper, hive-c0re already makes a *public*-TLS call: - # the OTEL exporter reaches whatever `otel.endpoint` names, typically a + # the OTEL exporter reaches whatever + # `deploy.swarm-otel.upstream.endpoint` names, typically a # normal internet host. The bundle is system CAs + the store's root # precisely so that path keeps working — narrowing it to the store's root # alone would fix a case nobody hits yet and break one that runs today. diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 6104581b..b162f8b8 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -8,8 +8,9 @@ # collector (./swarm-otel.nix). It holds no credential and picks no # destination: an agent's samples cross a hive boundary exactly once, and # what happens after that is the swarm's decision, not a hive's. The -# upstream options declared below describe that far end and are read one -# tier up — they stay here because they mean what they have always meant. +# upstream export is a swarm-collector deploy option, +# `services.hyperhive.deploy.swarm-otel.upstream` (./swarm-otel.nix); the +# old `otel.*` paths forward to it below. { pkgs, lib, @@ -56,6 +57,22 @@ in hostUnit = true; enable = runsHive; }) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "otel" "endpoint" ] + [ "services" "hyperhive" "deploy" "swarm-otel" "upstream" "endpoint" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "otel" "protocol" ] + [ "services" "hyperhive" "deploy" "swarm-otel" "upstream" "protocol" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "otel" "headersCredential" ] + [ "services" "hyperhive" "deploy" "swarm-otel" "upstream" "headersCredential" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "otel" "collector" "upstreamHeaderName" ] + [ "services" "hyperhive" "deploy" "swarm-otel" "upstream" "headerName" ] + ) (lib.mkRemovedOptionModule [ "services" "hyperhive" "otel" "enable" ] '' Every hive runs its own telemetry collector, and every agent exports to it by default, so there is no switch. Remove this definition. @@ -67,115 +84,6 @@ in ]; options.services.hyperhive.otel = { - endpoint = lib.mkOption { - type = lib.types.str; - default = ""; - example = "https://collector.example.com/otel"; - description = '' - Upstream OTLP endpoint: where telemetry ultimately goes, after it - has left the swarm. - - Read by the swarm's collector - ({option}`services.hyperhive.deploy.swarm-otel.enable`), which is the only - tier that holds the upstream credential. An agent is handed the - *first* hop instead — this hive's own collector — so this value is - never given to a container. - - Optional. Leave it empty and the swarm's own metrics store - ({option}`services.hyperhive.deploy.victoriametrics.enable`) is the - destination; that is a complete deployment, not a degraded one. - Set both and telemetry goes to both. - ''; - }; - - upstreamSignals = lib.mkOption { - type = lib.types.listOf ( - lib.types.enum [ - "metrics" - "logs" - ] - ); - default = [ - "metrics" - "logs" - ]; - example = [ "metrics" ]; - description = '' - Which signals the swarm's collector sends to - {option}`services.hyperhive.otel.endpoint`. The swarm's own metrics - and log stores always receive everything, whatever this lists. - - Use it when the upstream does not accept a signal, for example one - that answers 404 on `/v1/logs`: list only `"metrics"` and the logs - pipelines stop exporting upstream. An empty list defines no upstream - exporter at all. No pipeline exports traces, so they have no entry. - - Has no effect while {option}`services.hyperhive.otel.endpoint` is empty. - ''; - }; - - protocol = lib.mkOption { - type = lib.types.enum [ - "http/protobuf" - "http/json" - "grpc" - ]; - default = "http/protobuf"; - description = '' - OTLP wire protocol for the **upstream** link, honoured by the - swarm collector's exporter. - - Not what agents speak: their first hop is this hive's collector, - whose OTLP/HTTP receiver takes protobuf whatever the upstream - wants (see `hive-c0re/environment.nix`). - ''; - }; - - headersCredential = lib.mkOption { - # `str`, not `path`: a `path`-typed relative literal is hash-copied - # into the world-readable nix store at eval time, defeating the - # point. Keep it a string + require an absolute runtime path so the - # secret is only ever read from disk by systemd at start. - type = lib.types.nullOr lib.types.str; - default = null; - example = "/run/secrets/otel-headers"; - description = '' - Absolute path to an operator-provided secret file holding the - upstream auth header as `NAME=value` (e.g. - `Authorization=Bearer `). - - **Only the swarm's collector reads this** — the one tier that - talks to the upstream. It arrives as an `EnvironmentFile`, so the - value is never read by nix, never copied into the store or the - generated config, and never passed in argv; and it reaches - neither an agent container nor this hive's own collector, which - is the point of the tiers existing. Must be absolute. - - Leave null if the upstream needs no auth header; the collector - then sends none rather than an empty one. - ''; - }; - - collector.upstreamHeaderName = lib.mkOption { - type = lib.types.str; - default = "Authorization"; - description = '' - Name of the HTTP header the collector sends upstream, whose - *value* comes from `headersCredential`. - - The name is here and the value is not, and that split is forced - rather than chosen: the collector models exporter headers as a - static map, so rendering them means nix reading the value — the - one thing `headersCredential` being a path exists to prevent. - A name is public, a value is not. - - ⇒ exactly one header is expressible this way. A credential - carrying several (`a=1,b=2`) would be read as a single value, - which is why the shape is a named header rather than an opaque - blob: a second header has to be *declared*, not smuggled. - ''; - }; - collector.enable = lib.mkOption { type = lib.types.bool; readOnly = true; diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 16cc61a9..8016c5fc 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -17,7 +17,7 @@ let cfg = config.services.hyperhive.swarm.otel; deployCfg = config.services.hyperhive.deploy; - otelCfg = config.services.hyperhive.otel; + upstream = deployCfg.swarm-otel.upstream; vmCfg = config.services.hyperhive.swarm.victoriametrics; vlCfg = config.services.hyperhive.swarm.victorialogs; hyperhiveCfg = config.services.hyperhive; @@ -273,13 +273,13 @@ let storeSecretPath = "secret/swarm/services/${cfg.clientId}/oidc/client"; # The operator-configured upstream, named once: the same exporter carries - # every signal listed in `upstreamSignals`, so metrics and logs both reach + # every signal listed in `upstream.signals`, so metrics and logs both reach # it without a second definition. - upstreamExporterName = if otelCfg.protocol == "grpc" then "otlp" else "otlphttp"; + upstreamExporterName = if upstream.protocol == "grpc" then "otlp" else "otlphttp"; upstreamFor = signal: lib.optional ( - otelCfg.endpoint != "" && builtins.elem signal otelCfg.upstreamSignals + upstream.endpoint != "" && builtins.elem signal upstream.signals ) upstreamExporterName; # The collector refuses an exporter that no pipeline uses, so the exporter # is only defined when at least one signal goes to it. @@ -310,11 +310,126 @@ let storeRefreshTimer = import ./lib/store-refresh-timer.nix { }; in { - # What the collector IS to the swarm — the client it is registered as, where - # it exports — is `swarm.otel` in ./swarm-otel-service.nix. The secret is a - # path on the machine that runs it, so it hangs off the deployment. `enable` - # already lives in ./deploy.nix, which also carries the rename. + # What the collector IS to the swarm — the client it is registered as — is + # `swarm.otel` in ./swarm-otel-service.nix. The secret is a path on the + # machine that runs it, and the external export (`upstream`) is the + # operator's choice for this deployment, so both hang off the deployment. + # `enable` already lives in ./deploy.nix, which also carries the rename. options.services.hyperhive.deploy.swarm-otel = { + upstream = { + endpoint = lib.mkOption { + type = lib.types.str; + default = ""; + example = "https://collector.example.com/otel"; + description = '' + Upstream OTLP endpoint: where telemetry ultimately goes, after it + has left the swarm. + + Read by the swarm's collector + ({option}`services.hyperhive.deploy.swarm-otel.enable`), which is the only + tier that holds the upstream credential. An agent is handed the + *first* hop instead — this hive's own collector — so this value is + never given to a container. + + Optional. Leave it empty and the swarm's own metrics store + ({option}`services.hyperhive.deploy.victoriametrics.enable`) is the + destination; that is a complete deployment, not a degraded one. + Set both and telemetry goes to both. + ''; + }; + + signals = lib.mkOption { + type = lib.types.listOf ( + lib.types.enum [ + "metrics" + "logs" + ] + ); + default = [ + "metrics" + "logs" + ]; + example = [ "metrics" ]; + description = '' + Which signals the swarm's collector sends to + {option}`services.hyperhive.deploy.swarm-otel.upstream.endpoint`. The + swarm's own metrics and log stores always receive everything, + whatever this lists. + + Use it when the upstream does not accept a signal, for example one + that answers 404 on `/v1/logs`: list only `"metrics"` and the logs + pipelines stop exporting upstream. An empty list defines no upstream + exporter at all. No pipeline exports traces, so they have no entry. + + Has no effect while + {option}`services.hyperhive.deploy.swarm-otel.upstream.endpoint` is empty. + ''; + }; + + protocol = lib.mkOption { + type = lib.types.enum [ + "http/protobuf" + "http/json" + "grpc" + ]; + default = "http/protobuf"; + description = '' + OTLP wire protocol for the **upstream** link, honoured by the + swarm collector's exporter. + + Not what agents speak: their first hop is this hive's collector, + whose OTLP/HTTP receiver takes protobuf whatever the upstream + wants (see `hive-c0re/environment.nix`). + ''; + }; + + headersCredential = lib.mkOption { + # `str`, not `path`: a `path`-typed relative literal is hash-copied + # into the world-readable nix store at eval time, defeating the + # point. Keep it a string + require an absolute runtime path so the + # secret is only ever read from disk by systemd at start. + type = lib.types.nullOr lib.types.str; + default = null; + example = "/run/secrets/otel-headers"; + description = '' + Absolute path to an operator-provided secret file holding the + upstream auth header as `NAME=value` (e.g. + `Authorization=Bearer `). + + **Only the swarm's collector reads this** — the one tier that + talks to the upstream. It arrives as an `EnvironmentFile`, so the + value is never read by nix, never copied into the store or the + generated config, and never passed in argv; and it reaches + neither an agent container nor this hive's own collector, which + is the point of the tiers existing. Must be absolute. + + Leave null if the upstream needs no auth header; the collector + then sends none rather than an empty one. + ''; + }; + + headerName = lib.mkOption { + type = lib.types.str; + default = "Authorization"; + description = '' + Name of the HTTP header the collector sends upstream, whose + *value* comes from + {option}`services.hyperhive.deploy.swarm-otel.upstream.headersCredential`. + + The name is here and the value is not, and that split is forced + rather than chosen: the collector models exporter headers as a + static map, so rendering them means nix reading the value — the + one thing `headersCredential` being a path exists to prevent. + A name is public, a value is not. + + ⇒ exactly one header is expressible this way. A credential + carrying several (`a=1,b=2`) would be read as a single value, + which is why the shape is a named header rather than an opaque + blob: a second header has to be *declared*, not smuggled. + ''; + }; + }; + clientSecretFile = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; @@ -816,9 +931,9 @@ in isReadOnly = true; }; } - // lib.optionalAttrs (otelCfg.headersCredential != null) { - ${otelCfg.headersCredential} = { - hostPath = otelCfg.headersCredential; + // lib.optionalAttrs (upstream.headersCredential != null) { + ${upstream.headersCredential} = { + hostPath = upstream.headersCredential; isReadOnly = true; }; } @@ -853,7 +968,7 @@ in { ... }: { # This tier is the one that resolves an operator-configured - # hostname: `otel.endpoint` is an external URL, and reaching it + # hostname: `upstream.endpoint` is an external URL, and reaching it # is the entire reason this container holds a credential. The # `/etc/resolv.conf` nixos-containers copies in is a snapshot # taken once at boot, so without this the upstream export @@ -1076,16 +1191,16 @@ in }; } // lib.optionalAttrs haveUpstream { - ${if otelCfg.protocol == "grpc" then "otlp" else "otlphttp"} = { - endpoint = otelCfg.endpoint; + ${if upstream.protocol == "grpc" then "otlp" else "otlphttp"} = { + endpoint = upstream.endpoint; } - // lib.optionalAttrs (otelCfg.headersCredential != null) { + // lib.optionalAttrs (upstream.headersCredential != null) { # Interpolated by the collector from its environment at # runtime, never by nix: `EnvironmentFile` below is what # puts it there, so the value is not read into the store. - headers.${otelCfg.collector.upstreamHeaderName} = "\${env:${otelCfg.collector.upstreamHeaderName}}"; + headers.${upstream.headerName} = "\${env:${upstream.headerName}}"; } - // lib.optionalAttrs (otelCfg.protocol == "http/json") { encoding = "json"; }; + // lib.optionalAttrs (upstream.protocol == "http/json") { encoding = "json"; }; } // { # `logs_endpoint`, NOT `endpoint`: the latter is a base the @@ -1512,8 +1627,8 @@ in # EnvironmentFile format — so the secret reaches the process as # an environment variable without being read by nix, written to # the store, or passed in argv. - (lib.optionalAttrs (otelCfg.headersCredential != null) { - serviceConfig.EnvironmentFile = otelCfg.headersCredential; + (lib.optionalAttrs (upstream.headersCredential != null) { + serviceConfig.EnvironmentFile = upstream.headersCredential; }) # The other credential, and the other direction: the one above diff --git a/nix/module-eval/swarm-otel-core.nix b/nix/module-eval/swarm-otel-core.nix index 214b272a..697950bb 100644 --- a/nix/module-eval/swarm-otel-core.nix +++ b/nix/module-eval/swarm-otel-core.nix @@ -41,14 +41,28 @@ let deploy.victorialogs.enable = false; # An upstream makes `logExporterNames` longer than the store alone, which # is what the self-journal case below needs to tell the two apart. - otel.endpoint = "https://upstream.example/otel"; - otel.upstreamSignals = upstreamSignals; + deploy.swarm-otel.upstream.endpoint = "https://upstream.example/otel"; + deploy.swarm-otel.upstream.signals = upstreamSignals; }; otelNoStores = mkOtelNoStores [ "metrics" "logs" ]; + # A host config still on the `otel.*` upstream paths, every shipped one. + otelOldUpstreamPaths = hive { + deploy.swarm-otel.enable = true; + deploy.authelia.enable = true; + deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; + deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; + deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem"; + deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem"; + otel.endpoint = "https://old.example/otel"; + otel.protocol = "http/json"; + otel.headersCredential = "/run/secrets/old-otel-headers"; + otel.collector.upstreamHeaderName = "X-Old-Token"; + }; + # The same collector with the upstream limited to one signal, and with none. otelUpstreamMetricsOnly = mkOtelNoStores [ "metrics" ]; otelUpstreamNone = mkOtelNoStores [ ]; @@ -356,10 +370,33 @@ let && !(lib.any (n: builtins.elem "otlphttp" ps.${n}.exporters) (names "logs/")) && lib.all (n: builtins.elem "otlphttp/victorialogs" ps.${n}.exporters) (names "logs/"); } + { + # The control is `otelNoStores`: it sets the new paths and warns about + # none of the old ones, so the warnings below come from the renames. + name = "the old otel.* upstream paths set deploy.swarm-otel.upstream and warn"; + ok = + let + u = otelOldUpstreamPaths.services.hyperhive.deploy.swarm-otel.upstream; + warnsAbout = m: path: lib.any (lib.hasInfix "services.hyperhive.${path}") m.warnings; + in + u.endpoint == "https://old.example/otel" + && u.protocol == "http/json" + && u.headersCredential == "/run/secrets/old-otel-headers" + && u.headerName == "X-Old-Token" + && warnsAbout otelOldUpstreamPaths "otel.endpoint" + && warnsAbout otelOldUpstreamPaths "otel.protocol" + && warnsAbout otelOldUpstreamPaths "otel.headersCredential" + && warnsAbout otelOldUpstreamPaths "otel.collector.upstreamHeaderName" + && !(warnsAbout otelNoStores "otel.endpoint") + && + otelNoStores.services.hyperhive.deploy.swarm-otel.upstream.endpoint + == "https://upstream.example/otel" + && (otelSettings otelOldUpstreamPaths).exporters ? otlphttp; + } { # The collector rejects an exporter no pipeline uses, so with nothing # going upstream the exporter must not be defined. - name = "an empty upstreamSignals defines no upstream exporter and keeps the stores"; + name = "an empty upstream.signals defines no upstream exporter and keeps the stores"; ok = let s = otelSettings otelUpstreamNone;