363 lines
16 KiB
Nix
363 lines
16 KiB
Nix
# The swarm's telemetry collector as every hive sees it: the name it is
|
||
# reached on, its receiver ports, the published targets it scrapes, the client
|
||
# it is registered as and the audiences that client may present, identical on
|
||
# every host. What the host running it decides, and the container itself, are
|
||
# in ./swarm-otel.nix.
|
||
{
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
cfg = config.services.hyperhive.swarm.otel;
|
||
vmCfg = config.services.hyperhive.swarm.victoriametrics;
|
||
vlCfg = config.services.hyperhive.swarm.victorialogs;
|
||
hyperhiveCfg = config.services.hyperhive;
|
||
baoCfg = hyperhiveCfg.swarm.bao;
|
||
swarmDomain = hyperhiveCfg.swarm.domain;
|
||
|
||
# Total on a null swarm domain for the same reason every sibling module is:
|
||
# the required-domain assertion in hive-network.nix should be what an
|
||
# operator sees, not a coercion error from here.
|
||
domainBase = if swarmDomain == null then "invalid" else swarmDomain;
|
||
|
||
# Each store's OTLP route, by domain: the audiences `audience` below registers
|
||
# the client for. ⚠️ ./swarm-otel.nix binds the same strings for the
|
||
# exporters that request those tokens; the two copies must agree.
|
||
metricsPushUrl = "https://${vmCfg.domain}/opentelemetry/api/v1/push";
|
||
logsPushUrl = "https://${vlCfg.domain}/insert/opentelemetry/v1/logs";
|
||
|
||
pushAudiences = {
|
||
victoriametrics = metricsPushUrl;
|
||
victorialogs = logsPushUrl;
|
||
};
|
||
in
|
||
{
|
||
# `enable` is `services.hyperhive.deploy.swarm-otel.enable` — see ./deploy.nix.
|
||
# ⚠️ That is the SWARM collector. The per-hive one keeps its own
|
||
# `services.hyperhive.otel.collector.enable` (./otel.nix) and is a different
|
||
# option entirely — every hive runs that one.
|
||
options.services.hyperhive.swarm.otel = {
|
||
|
||
machine = lib.mkOption {
|
||
type = lib.types.str;
|
||
readOnly = true;
|
||
default = "swarm-otel";
|
||
description = ''
|
||
Name of the nixos-container this collector runs in — also the
|
||
`machinectl` name, so other modules may read it rather than
|
||
repeating the literal.
|
||
'';
|
||
};
|
||
|
||
telemetryPort = lib.mkOption {
|
||
type = lib.types.port;
|
||
default = 8889;
|
||
description = ''
|
||
Port this collector serves its **own** metrics on — queue depth,
|
||
refused and dropped samples, exporter failures. How you find out
|
||
that telemetry is being lost, so it is worth keeping rather than
|
||
switching off.
|
||
|
||
Bound on the container's own loopback, so no other collector
|
||
shares it; only this collector scrapes it.
|
||
'';
|
||
};
|
||
|
||
serviceProducers = lib.mkOption {
|
||
internal = true;
|
||
type = lib.types.attrsOf (
|
||
lib.types.submodule {
|
||
options = {
|
||
clientId = lib.mkOption {
|
||
type = lib.types.str;
|
||
description = ''
|
||
The producer's OAuth2 client id: the one audience its
|
||
receiver's `oidc/<name>` authenticator accepts, and the one
|
||
the forwarder requests.
|
||
'';
|
||
};
|
||
};
|
||
}
|
||
);
|
||
default = { };
|
||
description = ''
|
||
Swarm services whose own container forwards to this collector, keyed
|
||
by the route name: each gets a receiver `otlp/<name>`, an
|
||
authenticator `oidc/<name>` and a gateway location
|
||
`https://''${domain}/<name>/`, and its records land in the swarm
|
||
tier's pipelines with no `hive` label.
|
||
|
||
A receiver per producer because an `oidc` authenticator checks one
|
||
audience, and each service mints its token as its own client.
|
||
|
||
Declared below for every host, under no `enable`: the collector's
|
||
host renders the receiver whether or not the service runs there.
|
||
Every name must contain a fragment of
|
||
`nix/reserved-hive-fragments.nix`, so no hive can own its components.
|
||
'';
|
||
};
|
||
|
||
storeProducerName = lib.mkOption {
|
||
type = lib.types.str;
|
||
readOnly = true;
|
||
default = baoCfg.machine;
|
||
defaultText = lib.literalExpression "services.hyperhive.swarm.bao.machine";
|
||
description = ''
|
||
Path and component name of the store forwarder's own receiver
|
||
(`otlp/''${storeProducerName}`, `oidc/''${storeProducerName}`).
|
||
Published so `swarm-bao.nix` addresses the receiver by name rather
|
||
than repeating the string, exactly as `producerName` above is
|
||
published for `glue-controller-env.nix`.
|
||
|
||
Read-only and derived from the store's container name: two
|
||
spellings of a name both ends have to agree on is a 404 on a
|
||
request that authenticated perfectly.
|
||
'';
|
||
};
|
||
|
||
producerName = lib.mkOption {
|
||
type = lib.types.str;
|
||
readOnly = true;
|
||
default = "swarm";
|
||
description = ''
|
||
The swarm-tier producer's own component/path/authenticator name
|
||
— the same reserved literal this module's collector components
|
||
(`otlp/swarm`, `oidc/swarm`, `resource/swarm`, `metrics/swarm`)
|
||
are already named after internally. Published as an option so a
|
||
sibling module (`glue-controller-env.nix`) can address the receiver
|
||
by name — `https://''${domain}/''${producerName}/` — instead of
|
||
repeating the string. Read-only for the same reason
|
||
`clientId`/`machine`/`domain` are: two spellings of a name that
|
||
has to match on both ends is a mismatch waiting to happen.
|
||
'';
|
||
};
|
||
|
||
domain = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "otel.${domainBase}";
|
||
defaultText = lib.literalExpression ''"otel.''${services.hyperhive.swarm.domain}"'';
|
||
description = ''
|
||
Name the gateway serves this on. A sibling of the swarm's other
|
||
service names, so the swarm-services sub-CA can issue for it — see
|
||
`hive-tls.nix` for why a service name being a sibling rather than a
|
||
child decides which CA may sign it.
|
||
|
||
This is what the **hive** tier's exporter reaches — the hive
|
||
collector is a plain producer against this name exactly like every
|
||
other client of a swarm service, resolved locally by dnsmasq on a
|
||
co-located host and over the real network otherwise. There is no
|
||
separate loopback-vs-remote knob to get wrong: everything in this
|
||
swarm addresses its siblings by name.
|
||
'';
|
||
};
|
||
|
||
scrapeTargets = lib.mkOption {
|
||
type = lib.types.attrsOf lib.types.str;
|
||
default = { };
|
||
example = lib.literalExpression ''{ collector = "127.0.0.1:8889"; }'';
|
||
description = ''
|
||
Prometheus exposition endpoints this collector scrapes, as
|
||
`<job name> = "<host>:<port>"`. The collector runs in a network namespace of
|
||
its own, so `127.0.0.1` is the collector container's loopback: a target
|
||
here is a listener inside that container, such as its own telemetry.
|
||
|
||
A path and query may follow the port —
|
||
`"127.0.0.1:9100/stats?format=prometheus"` — for an
|
||
exporter that does not serve `/metrics`. Both are optional and
|
||
omitted when absent, so a bare `host:port` is scraped exactly as
|
||
before.
|
||
|
||
**A service declares its own entry, from its own module, under its
|
||
own `enable`.** Do not assemble the list here: an entry then exists
|
||
only where the service that named it runs, so a target is never
|
||
declared on a host that does not serve it. A swarm service on the
|
||
host is not reachable here, since its loopback is not the collector's:
|
||
it is a `publishedScrapeTargets` entry instead, which a collector on
|
||
any host scrapes.
|
||
|
||
⚠️ **That constrains the TARGET, not the SCRAPER, and the difference
|
||
is a silent gap.** Nothing here puts the collector on the same host.
|
||
Services are co-located by *default* — `swarm-required-services.nix`
|
||
derives their `enable` flags from one `lib.mkDefault` — not by
|
||
construction, and an operator may split them.
|
||
|
||
When they are split the target is simply never scraped: the
|
||
service's host declares an entry no local collector reads, and the
|
||
collector's host never enabled that service so has no entry at all.
|
||
No error, no eval failure, no warning.
|
||
|
||
**No assertion can catch this.** Two hosts are separate NixOS
|
||
evaluations with no shared context, so neither can see what the
|
||
other runs. A service that would be seriously wrong to lose should
|
||
say so in its own contribution, because saying it is the only
|
||
mechanism available.
|
||
|
||
Samples land in a swarm-level pipeline that stamps `swarm` and
|
||
**never** `hive`: a swarm service does not belong to a hive, and
|
||
`hive` stays a property of which authenticated receiver accepted a
|
||
push, not something a scrape can acquire.
|
||
|
||
Empty by default, in which case no scrape receiver, processor or
|
||
pipeline is emitted at all — an enabled scraper with nothing to
|
||
scrape is the inert configuration this option exists to avoid.
|
||
'';
|
||
};
|
||
|
||
publishedScrapeTargets = lib.mkOption {
|
||
type = lib.types.attrsOf lib.types.str;
|
||
default = { };
|
||
example = lib.literalExpression ''{ forge = "https://forge.example.com/metrics"; }'';
|
||
description = ''
|
||
Prometheus endpoints this collector scrapes **by name, with a
|
||
credential**, as `<job name> = "<url>"`.
|
||
|
||
**Entries are declared swarm-wide**: from this module's `config`,
|
||
unconditionally, keyed on the service's `swarm.<svc>.domain`, so
|
||
every host evaluates the same set. Unlike `scrapeTargets`, an entry
|
||
is not declared under the service's own `enable`: the collector
|
||
reads this list on its host and authelia's registration reads it on
|
||
authelia's host, and neither need be the service's host. A swarm
|
||
that does not run the service anywhere scrapes a name that answers
|
||
nothing, which shows as a failing job rather than a missing one.
|
||
|
||
The collector's OAuth2 client derives its permitted audiences from
|
||
these URLs, and authelia derives from each one the access rule that
|
||
admits only that client to it, so a target and the authorisation to
|
||
reach it are **one declaration**.
|
||
Two lists that must agree would be a drift to maintain, and its
|
||
failure mode is the bad one: a target whose audience was forgotten
|
||
authenticates against nothing and reads as a scrape failure rather
|
||
than a config mistake.
|
||
|
||
⚠️ A **full URL**, not `host:port`. Authelia validates a bearer
|
||
token against the address being requested, so the string here is
|
||
also the audience the token is minted for; a near miss (a trailing
|
||
slash, `http` for `https`) presents as a valid token rejected at
|
||
the target, several layers from its cause.
|
||
|
||
⚠️ Deliberately separate from `scrapeTargets`. An entry there is
|
||
trusted because the scraper and the target share a host — that
|
||
option is loopback-and-unauthenticated by contract. An entry here
|
||
is trusted because it presents a credential. One shape for both
|
||
would leave a reader unable to tell which of those a given target
|
||
relies on.
|
||
'';
|
||
};
|
||
|
||
clientId = lib.mkOption {
|
||
type = lib.types.str;
|
||
readOnly = true;
|
||
default = "swarm-collector";
|
||
description = ''
|
||
OAuth2 client id this collector authenticates as. Published so
|
||
authelia's `access_control` rules can name it without carrying a
|
||
second copy of the string, exactly as
|
||
`services.hyperhive.swarm.authelia.hiveClientPrefix` is published
|
||
for the queue's responder.
|
||
|
||
Two spellings drifting apart is not a build failure: authelia
|
||
refuses a rule naming an unregistered client in its startup
|
||
validator, so the swarm's SSO service fails to *restart* — long
|
||
after the change that caused it evaluated cleanly.
|
||
'';
|
||
};
|
||
|
||
audience = lib.mkOption {
|
||
type = lib.types.listOf lib.types.str;
|
||
readOnly = true;
|
||
default = lib.attrValues cfg.publishedScrapeTargets ++ lib.attrValues pushAudiences;
|
||
description = ''
|
||
Every audience this collector's OAuth2 client is permitted to
|
||
present a token for — the scrape targets it reads with a credential,
|
||
plus the stores it pushes to. Published read-only so
|
||
./glue-swarm-otel-oidc-client.nix can register the client wherever
|
||
authelia runs without restating the derivation: this option and
|
||
that registration are the same fact seen from two hosts, and a
|
||
second formula for it would be free to drift from this one.
|
||
'';
|
||
};
|
||
};
|
||
|
||
config.services.hyperhive.swarm.otel.serviceProducers = {
|
||
# The forwarder in ./swarm-bao.nix.
|
||
${cfg.storeProducerName} = {
|
||
clientId = baoCfg.otel.clientId;
|
||
};
|
||
# The forwarder in ./swarm-nats.nix, which addresses this route by the
|
||
# queue container's name.
|
||
swarm-nats = {
|
||
clientId = hyperhiveCfg.swarm.nats.clientId;
|
||
};
|
||
# The forwarders in ./swarm-builder.nix, ./swarm-claude-login.nix,
|
||
# ./swarm-notify-router.nix and ./swarm-review-classifier.nix, which
|
||
# address their routes by container name.
|
||
swarm-builder = {
|
||
clientId = hyperhiveCfg.swarm.builder.clientId;
|
||
};
|
||
swarm-claude-login = {
|
||
clientId = hyperhiveCfg.swarm.claude-login.clientId;
|
||
};
|
||
swarm-review-classifier = {
|
||
clientId = hyperhiveCfg.swarm.review-classifier.clientId;
|
||
};
|
||
swarm-notify-router = {
|
||
clientId = hyperhiveCfg.swarm.notify-router.clientId;
|
||
};
|
||
# The forwarders in ./swarm-ci-runner.nix, ./swarm-grafana.nix and
|
||
# ./swarm-victorialogs.nix, each authenticated as a client of its own.
|
||
swarm-ci-runner = {
|
||
clientId = hyperhiveCfg.swarm.ci-runner.otel.clientId;
|
||
};
|
||
swarm-grafana = {
|
||
clientId = hyperhiveCfg.swarm.grafana.otel.clientId;
|
||
};
|
||
swarm-victorialogs = {
|
||
clientId = hyperhiveCfg.swarm.victorialogs.otel.clientId;
|
||
};
|
||
# The forwarders in ./hive-forge/default.nix and ./hive-matrix.nix, each
|
||
# authenticated as a client of its own.
|
||
hive-forge = {
|
||
clientId = hyperhiveCfg.swarm.forge.otel.clientId;
|
||
};
|
||
hive-matrix = {
|
||
clientId = hyperhiveCfg.swarm.matrix.otel.clientId;
|
||
};
|
||
# The forwarder in ./swarm-victoriametrics.nix, which reaches this route
|
||
# through its host's nginx.
|
||
swarm-victoriametrics = {
|
||
clientId = hyperhiveCfg.swarm.victoriametrics.otel.clientId;
|
||
};
|
||
# The CI run poller in ./swarm-ci-metrics.nix, a host unit that pushes
|
||
# its run samples here directly.
|
||
swarm-ci-metrics = {
|
||
clientId = hyperhiveCfg.swarm.ci-metrics.clientId;
|
||
};
|
||
# The forwarder in ./swarm-authelia.nix.
|
||
swarm-authelia = {
|
||
clientId = hyperhiveCfg.swarm.authelia.otel.clientId;
|
||
};
|
||
};
|
||
|
||
# Not under any `enable`: see `publishedScrapeTargets` for why every host
|
||
# declares every entry.
|
||
#
|
||
# ⚠️ Each URL must be exactly what the collector requests, because authelia
|
||
# checks the token's audience against it: it has to agree with the gated
|
||
# `= /metrics` location on the service's vhost. A near miss is a correctly
|
||
# minted token refused at the target.
|
||
config.services.hyperhive.swarm.otel.publishedScrapeTargets = {
|
||
# The vhost location is in ./hive-forge/default.nix.
|
||
forgejo = "https://${hyperhiveCfg.swarm.forge.domain}/metrics";
|
||
# The vhost location is in ./swarm-authelia.nix.
|
||
authelia = "https://${hyperhiveCfg.swarm.authelia.domain}/metrics";
|
||
# The vhost location is in ./swarm-victoriametrics.nix.
|
||
victoriametrics = "https://${vmCfg.domain}/metrics";
|
||
# The vhost location is in ./swarm-victorialogs.nix.
|
||
victorialogs = "https://${hyperhiveCfg.swarm.victorialogs.domain}/metrics";
|
||
# The vhost location is in ./swarm-grafana.nix.
|
||
grafana = "https://${hyperhiveCfg.swarm.grafana.domain}/metrics";
|
||
# The vhost location is in ./swarm-nats.nix.
|
||
nats = "https://${hyperhiveCfg.swarm.nats.domain}/metrics";
|
||
};
|
||
}
|