d2fb4bff79
Drops swarm.otel.url (a loopback default an operator had to override on a split host) in favor of swarm.otel.domain -- the same gateway.localNames + nginx-vhost-through-the-gateway shape every other swarm service (authelia, grafana, victoriametrics, ui) already uses. The hive tier's exporter now reaches it as https://<domain> unconditionally, resolved locally by dnsmasq on a co-located host and over the real network otherwise, instead of a config knob nobody sets until they hit the silent drop. Costs CA trust on the hive tier: otel.nix wires lib/hive-ca-trust.nix's trustBundle with hostUnit = true on the opentelemetry-collector host unit, the same flag #3441/#3442 added for swarm-controller and hive-c0re. mara, #3125 comment 58363: "go c".
291 lines
13 KiB
Nix
291 lines
13 KiB
Nix
# The swarm's telemetry collector: one per swarm, in a `swarm-otel`
|
||
# nixos-container beside the swarm's other shared services.
|
||
#
|
||
# Two tiers, and they are separate on purpose:
|
||
#
|
||
# - `otel.nix` is the **hive** tier. It receives from this hive's agents
|
||
# on the bridge and forwards, and it holds no upstream credential.
|
||
# - this is the **swarm** tier. It is the only holder of the upstream
|
||
# credential, the only writer to the swarm's metrics store, and the
|
||
# place that will stamp `hive=` from the authenticated connection
|
||
# rather than from anything a sender can choose.
|
||
#
|
||
# On a host that runs both, both processes run. They are not collapsed:
|
||
# all-local is a statement about *where* processes run, not about what
|
||
# shape the deployment has, and a local tier boundary that disappears is
|
||
# one the local deployment stops testing. `hive=` attribution is the
|
||
# property that would differ, and the ingest auth that makes it
|
||
# unforgeable is built on this boundary existing.
|
||
#
|
||
# A container rather than a second host unit, for the same reason every
|
||
# sibling swarm service is one — and because `services.opentelemetry-collector`
|
||
# is a singleton NixOS option, already spoken for on the host by the hive
|
||
# tier. A container gets its own evaluation and therefore its own collector.
|
||
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
cfg = config.services.hyperhive.swarm.otel;
|
||
swarmCfg = config.services.hyperhive.swarm;
|
||
otelCfg = config.services.hyperhive.otel;
|
||
vmCfg = config.services.hyperhive.swarm.victoriametrics;
|
||
hyperhiveCfg = config.services.hyperhive;
|
||
gatewayCfg = hyperhiveCfg.gateway;
|
||
swarmDomain = hyperhiveCfg.swarm.domain;
|
||
|
||
# Total on a null swarm domain for the same reason every sibling module is:
|
||
# the required-domain assertion in hive-network.nix should be what an
|
||
# operator sees, not a coercion error from here.
|
||
domainBase = if swarmDomain == null then "invalid" else swarmDomain;
|
||
in
|
||
{
|
||
options.services.hyperhive.swarm.otel = {
|
||
enable = lib.mkOption {
|
||
type = lib.types.bool;
|
||
default = false;
|
||
description = ''
|
||
Run the swarm's telemetry collector on this host.
|
||
|
||
Asserted from `swarm.enableRequiredServices` in
|
||
./swarm-required-services.nix, with the metrics pair this
|
||
collector feeds: a swarm has one of these, and it belongs
|
||
wherever the shared services live rather than on every hive.
|
||
|
||
A hive that does not run it still runs its own hive-tier collector
|
||
(`services.hyperhive.otel.enable`) and reaches this one by name, at
|
||
{option}`services.hyperhive.swarm.otel.domain`.
|
||
'';
|
||
};
|
||
|
||
machine = lib.mkOption {
|
||
type = lib.types.str;
|
||
readOnly = true;
|
||
default = "swarm-otel";
|
||
description = ''
|
||
Name of the nixos-container this collector runs in — also the
|
||
`machinectl` name, so other modules may read it rather than
|
||
repeating the literal.
|
||
'';
|
||
};
|
||
|
||
port = lib.mkOption {
|
||
type = lib.types.port;
|
||
default = 4319;
|
||
description = ''
|
||
Port this collector's OTLP/HTTP receiver listens on.
|
||
|
||
⚠️ **Deliberately not 4318**, the OTLP/HTTP default, because the
|
||
hive tier already uses it (`services.hyperhive.otel.collector.port`)
|
||
and every swarm container shares the host's network namespace. Two
|
||
listeners claiming one port on one host is not a build failure —
|
||
it is a runtime coin toss over which one gets it, with nothing in
|
||
any log saying so. The same collision cost a release when grafana
|
||
and the forge both defaulted to 3000.
|
||
'';
|
||
};
|
||
|
||
telemetryPort = lib.mkOption {
|
||
type = lib.types.port;
|
||
default = 8889;
|
||
description = ''
|
||
Port this collector serves its **own** metrics on — queue depth,
|
||
refused and dropped samples, exporter failures. How you find out
|
||
that telemetry is being lost, so it is worth keeping rather than
|
||
switching off.
|
||
|
||
⚠️ **Deliberately not 8889's neighbour 8888**, which is the
|
||
collector's built-in default and therefore what the hive tier
|
||
already binds. Two collectors share a network namespace whenever
|
||
they are co-located, and unlike the OTLP port this one appears
|
||
nowhere in either config — it is a default inside the binary, so
|
||
nothing that compares configured ports can see the clash. The
|
||
second collector to start simply dies with
|
||
`bind: address already in use`.
|
||
'';
|
||
};
|
||
|
||
domain = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "otel.${domainBase}";
|
||
defaultText = lib.literalExpression ''"otel.''${services.hyperhive.swarm.domain}"'';
|
||
description = ''
|
||
Name the gateway serves this on. A sibling of the swarm's other
|
||
service names, so the swarm-services sub-CA can issue for it — see
|
||
`hive-tls.nix` for why a service name being a sibling rather than a
|
||
child decides which CA may sign it.
|
||
|
||
This is what the **hive** tier's exporter reaches — the hive
|
||
collector is a plain producer against this name exactly like every
|
||
other client of a swarm service, resolved locally by dnsmasq on a
|
||
co-located host and over the real network otherwise. There is no
|
||
separate loopback-vs-remote knob to get wrong: `swarm-nats` is the
|
||
deliberate exception to this pattern (its cross-hive reach is the
|
||
wireguard mesh, not the gateway), everything else in this swarm
|
||
addresses its siblings by name.
|
||
'';
|
||
};
|
||
};
|
||
|
||
config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) {
|
||
# The gateway name, inside `cfg.enable` — that guard is the load-bearing
|
||
# part. Every hive in a swarm may know this collector exists, but only
|
||
# the host that RUNS it may claim the name; a client hive declaring the
|
||
# vhost would answer for a service it does not have.
|
||
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
||
|
||
# OTLP/HTTP, not a browsable UI, but the same reverse-proxy shape as
|
||
# every sibling swarm service: TLS terminates here, then plain http to
|
||
# the co-located container over loopback (shared netns, like the store
|
||
# this collector writes to).
|
||
services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // {
|
||
listen = gatewayCfg.lib.listen;
|
||
extraConfig = gatewayCfg.lib.securityHeaders;
|
||
locations."/" = {
|
||
proxyPass = "http://127.0.0.1:${toString cfg.port}";
|
||
};
|
||
};
|
||
|
||
assertions = [
|
||
{
|
||
# The tier exists to hold the upstream credential and to write the
|
||
# swarm's store. With neither, it is a process that receives
|
||
# samples and drops them — which looks healthy and loses data.
|
||
assertion = otelCfg.endpoint != "" || vmCfg.enable;
|
||
message = ''
|
||
services.hyperhive.swarm.otel.enable is true but this collector
|
||
has nowhere to send what it receives:
|
||
services.hyperhive.otel.endpoint is empty and
|
||
services.hyperhive.swarm.victoriametrics.enable is false.
|
||
|
||
Set the endpoint to export upstream, or enable the swarm's
|
||
metrics store.
|
||
'';
|
||
}
|
||
];
|
||
|
||
containers.${cfg.machine} = {
|
||
autoStart = true;
|
||
ephemeral = false;
|
||
# Shared host netns, like every sibling swarm service: the hive tier
|
||
# reaches this collector, and this collector reaches the metrics
|
||
# store, without either crossing a network boundary that would need
|
||
# its own trust material.
|
||
privateNetwork = false;
|
||
|
||
# The upstream credential is operator-provided and lives on the host.
|
||
# Read-only, and only when one is configured — binding a path that
|
||
# does not exist makes nixos-container refuse to start the container,
|
||
# which is a stall several layers from its cause.
|
||
bindMounts = lib.optionalAttrs (otelCfg.headersCredential != null) {
|
||
${otelCfg.headersCredential} = {
|
||
hostPath = otelCfg.headersCredential;
|
||
isReadOnly = true;
|
||
};
|
||
};
|
||
|
||
config =
|
||
{ ... }:
|
||
{
|
||
# This tier is the one that resolves an operator-configured
|
||
# hostname: `otel.endpoint` is an external URL, and reaching it
|
||
# is the entire reason this container holds a credential. The
|
||
# `/etc/resolv.conf` nixos-containers copies in is a snapshot
|
||
# taken once at boot, so without this the upstream export
|
||
# depends on the host's file having been right at that instant.
|
||
imports = [
|
||
(import ./swarm-container-resolver.nix {
|
||
inherit (config.services.hyperhive.network) bridgeIp;
|
||
dnsConsumers = [ "opentelemetry-collector.service" ];
|
||
})
|
||
];
|
||
|
||
system.stateVersion = config.system.stateVersion;
|
||
networking.firewall.enable = false;
|
||
# Keep the host-copied /etc/resolv.conf intact — same reasoning
|
||
# as the sibling swarm containers.
|
||
networking.resolvconf.enable = lib.mkForce false;
|
||
|
||
services.opentelemetry-collector = {
|
||
enable = true;
|
||
package = pkgs.opentelemetry-collector-contrib;
|
||
# Runs `otelcol validate` at build time. ⚠️ A parser, not a
|
||
# wiring check: it accepts a receiver naming an absent
|
||
# extension, and the collector then dies at startup. A green
|
||
# build does not prove this config starts, never mind that a
|
||
# sample arrives — which is why this module's gate pushes a
|
||
# real sample through both tiers into the store.
|
||
validateConfigFile = true;
|
||
settings = {
|
||
receivers.otlp.protocols.http.endpoint = "127.0.0.1:${toString cfg.port}";
|
||
|
||
exporters =
|
||
lib.optionalAttrs vmCfg.enable {
|
||
# `metrics_endpoint`, NOT `endpoint`: the latter is a
|
||
# base that otlphttp appends `/v1/metrics` to, while
|
||
# VictoriaMetrics serves OTLP at
|
||
# `/opentelemetry/api/v1/push`. With `endpoint` the
|
||
# collector answers 200 to its own clients and posts the
|
||
# samples to a path that does not exist. Measured
|
||
# end-to-end, not read — `state/probe-3265-collector-to-vm.sh`.
|
||
"otlphttp/victoriametrics".metrics_endpoint =
|
||
"http://127.0.0.1:${toString vmCfg.port}/opentelemetry/api/v1/push";
|
||
}
|
||
// lib.optionalAttrs (otelCfg.endpoint != "") {
|
||
${if otelCfg.protocol == "grpc" then "otlp" else "otlphttp"} = {
|
||
endpoint = otelCfg.endpoint;
|
||
}
|
||
// lib.optionalAttrs (otelCfg.headersCredential != null) {
|
||
# Interpolated by the collector from its environment at
|
||
# runtime, never by nix: `EnvironmentFile` below is what
|
||
# puts it there, so the value is not read into the store.
|
||
headers.${otelCfg.collector.upstreamHeaderName} = "\${env:${otelCfg.collector.upstreamHeaderName}}";
|
||
}
|
||
// lib.optionalAttrs (otelCfg.protocol == "http/json") { encoding = "json"; };
|
||
};
|
||
|
||
# Moves this collector's self-metrics off the built-in
|
||
# default of `localhost:8888`, which the hive tier holds.
|
||
#
|
||
# ⚠️ `metrics.address` is the spelling that looks right and
|
||
# is REJECTED by this collector version — measured, not
|
||
# read: `'migration.MetricsConfigV030' has invalid keys:
|
||
# address`. `readers` is the schema it accepts, and the
|
||
# difference is a startup failure rather than a warning.
|
||
service.telemetry.metrics.readers = [
|
||
{
|
||
pull.exporter.prometheus = {
|
||
host = "127.0.0.1";
|
||
port = cfg.telemetryPort;
|
||
};
|
||
}
|
||
];
|
||
|
||
service.pipelines.metrics = {
|
||
receivers = [ "otlp" ];
|
||
# Fan-out, not a choice: with both configured the same
|
||
# samples go upstream AND into the swarm's store. The store
|
||
# is for looking at this swarm; the upstream is for whoever
|
||
# aggregates across swarms, and neither replaces the other.
|
||
exporters =
|
||
lib.optional (otelCfg.endpoint != "") (if otelCfg.protocol == "grpc" then "otlp" else "otlphttp")
|
||
++ lib.optional vmCfg.enable "otlphttp/victoriametrics";
|
||
};
|
||
};
|
||
};
|
||
|
||
# The credential file is already `NAME=value`, systemd's
|
||
# EnvironmentFile format — so the secret reaches the process as an
|
||
# environment variable without being read by nix, written to the
|
||
# store, or passed in argv.
|
||
systemd.services.opentelemetry-collector.serviceConfig =
|
||
lib.optionalAttrs (otelCfg.headersCredential != null)
|
||
{
|
||
EnvironmentFile = otelCfg.headersCredential;
|
||
};
|
||
};
|
||
};
|
||
};
|
||
}
|