nix: the store's own collector scrapes its metrics listener
bao's metrics were scraped by the SWARM collector over loopback, via a `swarm.otel.scrapeTargets.bao` entry gated on `deploy.swarm-otel.enable` — "does the swarm's collector run on THIS host". It had to be: loopback only reaches a reader that landed on the same host. What that rendered everywhere else was nothing at all. Off that host the metrics listener was not emitted, so the store's metrics reached the store nowhere, and a host with no entry is indistinguishable from a host nobody asked to scrape. Moves the scrape into the collector this container already runs, per mara on #4537: "move the existing scraper to the local collector". The container shares the host netns (privateNetwork = false), so the scrape still dials 127.0.0.1 — the listener keeps its address, its `metrics_only` narrowing and its loopback-only bind, and the API listener's `tls_require_and_verify_client_cert` is untouched. The listener and its `prometheus_retention_time` lose their gate: the reader ships with the store now, so there is no host where the endpoint has none. The metrics pipeline reuses the logs pipeline's `resource` processor and `otlphttp` exporter, so both signals carry the same `service.name` and leave by the one hop. Logs are unaffected: `journaldUnits` and --link-journal=host stay until every sibling swarm container has a collector of its own. The module-eval absence arm "a store with no collector beside it serves no metrics" is inverted rather than dropped — the condition it asserted is the bug. Three cases join it: the job is in swarm-bao AND gone from swarm-otel (a move, not a copy), the scrape target and listener are both pinned to loopback, and the metrics pipeline shares its exporter with the logs one.
This commit is contained in:
atlas
mara
parent
d4313fc34d
commit
bb0afcd256
8 changed files with 815 additions and 58 deletions
|
|
@ -30,6 +30,7 @@
|
|||
./glue-matrix-ctl-bao-identity.nix
|
||||
./glue-queue-agent-credential.nix
|
||||
./glue-secret-publisher-bao-identity.nix
|
||||
./glue-swarm-bao-otel-oidc-client.nix
|
||||
./glue-swarm-otel-oidc-client.nix
|
||||
./swarm-authelia.nix
|
||||
./swarm-bao.nix
|
||||
|
|
|
|||
56
nix/host-modules/glue-swarm-bao-otel-oidc-client.nix
Normal file
56
nix/host-modules/glue-swarm-bao-otel-oidc-client.nix
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
# Glue: register the secret store's own journal forwarder as an OIDC client
|
||||
# wherever authelia runs.
|
||||
#
|
||||
# ONE PAIRING PER FILE — swarm-bao's forwarder ← authelia, and nothing else.
|
||||
# Deleting this leaves a forwarder authelia has never heard of: the token
|
||||
# endpoint refuses it, nothing is minted, the publisher has nothing to copy
|
||||
# into the store, and the collector's export is a 401 nobody asked for.
|
||||
#
|
||||
# ⚠️ A SECOND client beside ./glue-swarm-otel-oidc-client.nix's, not a reuse
|
||||
# of it. That one is the swarm collector's identity for what IT pushes; this
|
||||
# one belongs to the collector inside the store's container, which is a
|
||||
# different principal on a different host — one identity per principal, the
|
||||
# rule `swarm-controller.nix` states over its own `queueClientId`.
|
||||
#
|
||||
# ⚠️ Gated on authelia being HERE, and deliberately NOT on this host running
|
||||
# the store. A client is a row in THIS host's provider config, so it can only
|
||||
# be declared where that config is rendered, and ./swarm-bao.nix's `config`
|
||||
# hangs off `deploy.bao.enable` — the split ./glue-grafana-oidc-client.nix and
|
||||
# ./glue-swarm-otel-oidc-client.nix each made for the same reason. Read either
|
||||
# file's own comment for the property this one shares with them.
|
||||
{
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
let
|
||||
hyperhiveCfg = config.services.hyperhive;
|
||||
deployCfg = hyperhiveCfg.deploy;
|
||||
baoCfg = hyperhiveCfg.swarm.bao;
|
||||
in
|
||||
{
|
||||
config = lib.mkIf (hyperhiveCfg.enable && deployCfg.authelia.enable) {
|
||||
# One declaration, two readers: `clientId` is a read-only option
|
||||
# ./swarm-bao.nix owns, and ./swarm-otel.nix builds this principal's
|
||||
# authenticator audience from the same option.
|
||||
services.hyperhive.swarm.authelia.oidc.clients = [
|
||||
{
|
||||
id = baoCfg.otel.clientId;
|
||||
description = "HyperHive secret store journal forwarder";
|
||||
kind = "machine";
|
||||
redirectUris = [ ];
|
||||
# Its own id as its own permitted audience — the self-referential
|
||||
# form `swarm-authelia.nix`'s `hiveClients` uses, which is the shape
|
||||
# to copy here: this forwarder's token lands on an `oidc/*` receiver
|
||||
# of the swarm collector, exactly as a hive's does.
|
||||
audience = [ baoCfg.otel.clientId ];
|
||||
# ⚠️ What makes the token READABLE by that receiver at all. Authelia's
|
||||
# default is an opaque handle, and an `oidc` extension verifies
|
||||
# offline against `/jwks.json` — so without this the export fails with
|
||||
# a message about the verifier rather than about the token. Same
|
||||
# value, same reason, as every hive's client.
|
||||
accessTokenSignedResponseAlg = "RS256";
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
|
|
@ -429,19 +429,23 @@ let
|
|||
// metricsListener;
|
||||
|
||||
# Metrics get their own listener rather than a flag on the one above, and
|
||||
# that follows from what a scraper can express: `swarm.otel.scrapeTargets`
|
||||
# carries no scheme and no credential, while the API listener is TLS and —
|
||||
# once a client CA is set — demands a client certificate. The collector
|
||||
# cannot reach it at all.
|
||||
# that follows from what a scraper can express: a prometheus scrape config
|
||||
# carries no client certificate, while the API listener is TLS and — once a
|
||||
# client CA is set — demands one. A scraper cannot reach it at all.
|
||||
#
|
||||
# `metrics_only` narrows this one to the metrics path (every other path 404s)
|
||||
# and the unauthenticated access is confined to loopback. **Deliberately
|
||||
# unauthenticated for now** — a tracked follow-up owns giving the collector a
|
||||
# credential, since no scrape option can carry one today.
|
||||
# unauthenticated** — the only process that can dial 127.0.0.1 here is one
|
||||
# already inside this netns, which is the container's own collector and the
|
||||
# store itself. Widening this address, or dropping
|
||||
# `tls_require_and_verify_client_cert` on the API listener to scrape THAT
|
||||
# instead, would each trade that property away for nothing.
|
||||
#
|
||||
# Exists only where a collector does: an endpoint with no reader is exposure
|
||||
# bought for nothing.
|
||||
metricsListener = lib.optionalAttrs scrapeHere {
|
||||
# Unconditional, and that is the change this carries: the reader is now the
|
||||
# collector in this very container (below), which renders wherever the store
|
||||
# does. There is no longer a host where this endpoint has no reader, so there
|
||||
# is nothing left for it to be a function of.
|
||||
metricsListener = {
|
||||
metrics = {
|
||||
type = "tcp";
|
||||
address = "127.0.0.1:${toString baoDeploy.metricsPort}";
|
||||
|
|
@ -453,21 +457,32 @@ let
|
|||
};
|
||||
};
|
||||
|
||||
scrapeHere = deployCfg.swarm-otel.enable;
|
||||
# What the container's own collector dials. Loopback inside a netns this
|
||||
# container SHARES with the host (`privateNetwork = false` below), so the
|
||||
# scrape crosses no network and needs no bind address wider than the one
|
||||
# above.
|
||||
metricsScrapeTarget = "127.0.0.1:${toString baoDeploy.metricsPort}";
|
||||
|
||||
# ⚠️ The forwarder below has NO condition of its own, and `scrapeHere` above
|
||||
# is not a template for one — the two are different tiers on purpose. A
|
||||
# loopback metrics listener works only where the scraper is, which is what
|
||||
# `Here` in that name says; forwarding reaches the swarm's collector BY NAME,
|
||||
# from wherever this store runs.
|
||||
# ⚠️ Neither the listener above nor the forwarder below has a condition of
|
||||
# its own, and they lost theirs for the same reason at different times.
|
||||
#
|
||||
# There is nothing left to gate on. "Does the swarm have a collector" is not
|
||||
# a question that can be false (mara: "the swarm always has at least one
|
||||
# instance of all the swarm level services"), so both earlier gates —
|
||||
# `otel.enable`, then `deploy.swarm-otel.enable` — asked about the wrong
|
||||
# host and rendered no forwarder wherever the answer happened to be no. The
|
||||
# only condition that belongs here is the one already enclosing this block:
|
||||
# the store is deployed on this host.
|
||||
# The listener used to be gated on `scrapeHere = deployCfg.swarm-otel.enable`
|
||||
# — "does the swarm's collector happen to run on THIS host" — because the
|
||||
# thing reading it was that collector, over loopback, which only works when
|
||||
# the two land together. mara's ruling (*"move the existing scraper
|
||||
# to the local collector"*) removes the premise rather than the symptom: the
|
||||
# scraper is now in this container, so the reader is wherever the store is,
|
||||
# and the metrics path stops being silently dead on every host that does not
|
||||
# also run swarm-otel.
|
||||
#
|
||||
# The forwarder never had a defensible one. "Does the swarm have a collector"
|
||||
# is not a question that can be false (mara: "the swarm always has at least
|
||||
# one instance of all the swarm level services"), so both earlier gates —
|
||||
# `otel.enable`, then `deploy.swarm-otel.enable` — asked about the wrong host
|
||||
# and rendered no forwarder wherever the answer happened to be no.
|
||||
#
|
||||
# The only condition that belongs on either is the one already enclosing this
|
||||
# block: the store is deployed on this host.
|
||||
|
||||
# The swarm collector's own name, served by the gateway — the address
|
||||
# `swarm.otel.domain` exists to be, resolved by dnsmasq on a co-located host
|
||||
|
|
@ -479,7 +494,49 @@ let
|
|||
# The port suffix follows ./hive-forge/default.nix: elided on the canonical
|
||||
# 443 so this renders byte for byte like the tier's other two consumers of
|
||||
# this name, present when an operator moved the gateway's TLS port.
|
||||
otelFirstHop = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${swarmOtelCfg.producerName}";
|
||||
otelFirstHop = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${swarmOtelCfg.storeProducerName}";
|
||||
|
||||
# The forwarder's own credential, and the three names it takes on the way
|
||||
# in. The shape is ./swarm-otel.nix's `swarm-bao-otel-oidc` — the unit that
|
||||
# already reads an OIDC client secret out of the store with this host's
|
||||
# certificate and lands it in a collector's container — because that route
|
||||
# is proven and there is no second one worth inventing.
|
||||
#
|
||||
# Where the publisher on authelia's host leaves it. The `services` segment
|
||||
# is `swarm-secret-client`'s `path::Kind::Service`: a swarm service's client
|
||||
# is registered once for the whole swarm, so its secret is one value. The
|
||||
# prefix is also exactly what a hive certificate's read grant covers, so a
|
||||
# path outside it answers 403 however correct it looks.
|
||||
forwarderStoreSecretPath = "secret/swarm/services/${cfg.otel.clientId}/oidc/client";
|
||||
# At rest in the container's tree, written by the host unit below — under
|
||||
# /var/lib rather than /run, because a secret that evaporates on reboot
|
||||
# turns a working export into an intermittent one.
|
||||
forwarderSecretInContainer = "/var/lib/swarm-bao-otel-oidc/${cfg.otel.clientId}.secret";
|
||||
forwarderHostSecretPath = "/var/lib/nixos-containers/${cfg.machine}${forwarderSecretInContainer}";
|
||||
forwarderHostSecretDir = builtins.dirOf forwarderHostSecretPath;
|
||||
# The collector runs under `DynamicUser` and opens `client_secret_file`
|
||||
# itself, so there is no uid to hand a 0400 file to. `LoadCredential` reads
|
||||
# it as root before the sandbox exists and re-exposes it under a path that
|
||||
# does not depend on which uid the unit got; the collector expands
|
||||
# `${env:CREDENTIALS_DIRECTORY}` at load, so this id is the only spelling.
|
||||
forwarderCredentialId = "oidc-client-secret";
|
||||
# Named once and read three times — the extension that defines it, the
|
||||
# exporter that names it, and `service.extensions`, where an omission makes
|
||||
# it inert rather than an error.
|
||||
forwarderAuthName = "oauth2client/${swarmOtelCfg.storeProducerName}";
|
||||
autheliaCfg = hyperhiveCfg.swarm.authelia;
|
||||
|
||||
# Outbound TLS for that collector: it dials the swarm collector's gateway
|
||||
# name and authelia's token endpoint, both leaves of the runtime-generated
|
||||
# hive CA, which cannot be baked into a derivation. Same helper, same two
|
||||
# halves and the same `SSL_CERT_FILE` consumption as ./swarm-otel.nix's own
|
||||
# container — a Go process whose trust variable REPLACES the store, which is
|
||||
# what the bundle (system CAs ++ our anchors) is for.
|
||||
caTrust = import ./lib/hive-ca-trust.nix {
|
||||
inherit lib;
|
||||
tlsCfg = deployCfg.hive-controller.tls;
|
||||
gatewayCfg = hyperhiveCfg.gateway;
|
||||
};
|
||||
|
||||
swarmOtelCfg = hyperhiveCfg.swarm.otel;
|
||||
gatewayPortSuffix =
|
||||
|
|
@ -490,8 +547,9 @@ let
|
|||
|
||||
# Non-zero is what SERVES the endpoint at all — the switch is a duration, not
|
||||
# a boolean, so a zero here is an openbao that answers 404 on a listener
|
||||
# configured to do nothing else.
|
||||
telemetry = lib.optionalAttrs scrapeHere {
|
||||
# configured to do nothing else. Unconditional for the same reason the
|
||||
# listener above is: its reader ships with the store now.
|
||||
telemetry = {
|
||||
telemetry = {
|
||||
prometheus_retention_time = "24h";
|
||||
disable_hostname = true;
|
||||
|
|
@ -885,6 +943,29 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
otel.clientId = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
readOnly = true;
|
||||
default = "swarm-bao-collector";
|
||||
description = ''
|
||||
OAuth2 client id the collector inside the store's container
|
||||
authenticates as, and — self-referentially, the shape every hive's
|
||||
client already uses — the audience it asks its token for.
|
||||
|
||||
**Its own, not the swarm collector's and not
|
||||
`swarm-controller`'s.** One identity per principal: this forwarder
|
||||
runs wherever the store runs, which is not where either of those
|
||||
two runs, and the receiver it pushes to
|
||||
(`swarm.otel.storeProducerName`) admits this id alone.
|
||||
|
||||
Swarm-wide and read-only because three hosts have to agree on it:
|
||||
authelia registers the client
|
||||
(`glue-swarm-bao-otel-oidc-client.nix`), the swarm collector checks
|
||||
the audience (`swarm-otel.nix`), and the store's host reads the
|
||||
minted secret back out of the store under a path composed from it.
|
||||
Two spellings present as a healthy-looking 401.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
# ⚠️ Gated on `deploy.bao.enable`, and that is load-bearing rather than
|
||||
|
|
@ -973,15 +1054,21 @@ in
|
|||
"openbao"
|
||||
"swarm-bao-certs"
|
||||
"swarm-bao-token"
|
||||
"swarm-bao-forwarder-oidc"
|
||||
];
|
||||
|
||||
services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf scrapeHere {
|
||||
# Path and query, not just `host:port`: openbao serves no `/metrics`
|
||||
# at all, and `/v1/sys/metrics` answers JSON unless the format is
|
||||
# asked for. A scrape of the default path 404s, which reads as a
|
||||
# dead exporter rather than a wrong address.
|
||||
bao = "127.0.0.1:${toString baoDeploy.metricsPort}/v1/sys/metrics?format=prometheus";
|
||||
};
|
||||
# 🚫 No `swarm.otel.scrapeTargets.bao` entry any more, and its absence is
|
||||
# the deliverable rather than a tidy-up. That option is read only by the
|
||||
# SWARM collector, over loopback, so declaring the store there meant its
|
||||
# metrics existed exactly on the hosts that also ran swarm-otel and
|
||||
# nowhere else — silently, since a store with no entry looks identical to
|
||||
# one nothing scrapes. The scrape moved into this container's own
|
||||
# collector (see `receivers.prometheus` below), which travels with the
|
||||
# store.
|
||||
#
|
||||
# Logs are untouched by that move: `journaldUnits` above still names this
|
||||
# store's units for the shared collector, and the two are retired
|
||||
# together once every sibling swarm container has a collector of its own.
|
||||
|
||||
# ⚠️ The one nginx exception to this file's header, and it is one because
|
||||
# it never terminates. `ssl_preread` reads the SNI off the ClientHello
|
||||
|
|
@ -1015,6 +1102,113 @@ in
|
|||
# still proves nothing — bao answers nothing without a client
|
||||
# certificate its CA signed.
|
||||
services.hyperhive.network.exposeHostPorts = [ cfg.port ];
|
||||
|
||||
# THE delivery unit for the forwarder's own OIDC client secret, copied
|
||||
# in shape from ./swarm-otel.nix's `swarm-bao-otel-oidc`: a cert login
|
||||
# that fails LOUDLY, since every state it fails on is one a retry fixes,
|
||||
# then a read, then a root-owned file in the container's tree that
|
||||
# `LoadCredential` re-exposes to the collector's dynamic user.
|
||||
#
|
||||
# ⚠️ ORDERED AFTER the store's container and deliberately NOT before it,
|
||||
# which is the one line that differs from the unit it copies. That one
|
||||
# reads a store in the container NEXT DOOR; this one reads the store in
|
||||
# the very container it is delivering into, so "before" is a wait on a
|
||||
# process that cannot start until this finishes. There is no bootstrap
|
||||
# cycle behind it — the identity used here is this host's static
|
||||
# `swarm-bao-pki` certificate, not anything the store mints — only an
|
||||
# ordering one, and the cost is bounded: the file is under /var/lib, so
|
||||
# it survives reboots and only a FIRST boot has the collector starting
|
||||
# before it exists. `LoadCredential` refuses to start a unit whose
|
||||
# source is missing, so that boot is a collector that restarts, says so
|
||||
# each time, and comes up the moment this lands. Loud and self-healing
|
||||
# rather than silently exporting without a credential.
|
||||
systemd.services.swarm-bao-forwarder-oidc = {
|
||||
description = "fetch the secret store forwarder's OIDC client secret from the store";
|
||||
after = [
|
||||
"swarm-bao-pki.service"
|
||||
"container@${cfg.machine}.service"
|
||||
];
|
||||
wants = [ "container@${cfg.machine}.service" ];
|
||||
requires = [ "swarm-bao-pki.service" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
path = [
|
||||
baoDeploy.package
|
||||
pkgs.coreutils
|
||||
];
|
||||
# Sized for the race this loses: the secret is minted by authelia and
|
||||
# copied in by the publisher, both of which may be a host away and
|
||||
# neither of which this boot waits on. `StartLimit*` are `[Unit]`
|
||||
# settings — systemd ignores them under `[Service]` — and the window
|
||||
# has to exceed `RestartSec × burst`.
|
||||
startLimitBurst = 4;
|
||||
startLimitIntervalSec = 300;
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
SyslogIdentifier = "swarm-bao-forwarder-oidc";
|
||||
TimeoutStartSec = 30;
|
||||
Restart = "on-failure";
|
||||
RestartSec = 15;
|
||||
};
|
||||
environment = {
|
||||
BAO_ADDR = "https://${cfg.domain}:${toString cfg.port}";
|
||||
BAO_CLIENT_CERT = baoDeploy.clientCertFile;
|
||||
BAO_CLIENT_KEY = baoDeploy.clientKeyFile;
|
||||
}
|
||||
# Absent means the system trust store, which is what a deployment with
|
||||
# a real CA wants and what a self-signed one must not be left with.
|
||||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||||
BAO_CACERT = baoDeploy.serverCaFile;
|
||||
};
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
# `bao`'s own message is the only thing separating a missing value
|
||||
# from a refused identity from an unreachable store, and this unit
|
||||
# retries on all three — so it reports which one rather than
|
||||
# asserting all three in a sentence of ours.
|
||||
err="$(mktemp)"
|
||||
trap 'rm -f "$err"' EXIT
|
||||
|
||||
# Cert auth is a login, not a transport setting: the `BAO_CLIENT_*`
|
||||
# variables only decide which certificate the handshake presents.
|
||||
# Without a token `bao` asks its token helper, and that is a `sh`
|
||||
# this unit's `path` does not carry — `-token-only` answers on
|
||||
# stdout and skips the helper on both sides.
|
||||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||||
echo "could not log in to the swarm secret store with this host's certificate." >&2
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
export BAO_TOKEN
|
||||
|
||||
# ⚠️ Non-zero, where the sibling unit one module over exits 0 on this
|
||||
# branch. That collector treats a missing secret as a supported
|
||||
# degrade; this one has no such mode — the receiver it pushes to
|
||||
# takes an authenticated request and nothing else — so "not
|
||||
# published yet" is a state to retry out of, not to settle into.
|
||||
if ! secret="$(bao kv get -field=value ${lib.escapeShellArg forwarderStoreSecretPath} 2>"$err")"; then
|
||||
echo "the store did not return ${forwarderStoreSecretPath}: the forwarder has no client secret yet." >&2
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$secret" ]; then
|
||||
echo "the store returned an empty ${forwarderStoreSecretPath}." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# root-owned 0400, written with a shell builtin and never handed to
|
||||
# a program: `printf` is bash's own, so the plaintext never becomes
|
||||
# an argument in /proc the way `install <<<"$secret"` or an `echo`
|
||||
# from `path` would.
|
||||
install -d -m 0755 ${lib.escapeShellArg forwarderHostSecretDir}
|
||||
umask 077
|
||||
printf '%s\n' "$secret" > ${lib.escapeShellArg forwarderHostSecretPath}
|
||||
chown root:root ${lib.escapeShellArg forwarderHostSecretPath}
|
||||
chmod 0400 ${lib.escapeShellArg forwarderHostSecretPath}
|
||||
'';
|
||||
};
|
||||
})
|
||||
|
||||
(lib.mkIf (hyperhiveCfg.enable && deployCfg.bao.enable && haveServerTls) {
|
||||
|
|
@ -1399,6 +1593,11 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
# The CA bind source is written at runtime by a host unit, so the
|
||||
# container has to start after it — otherwise nspawn sets up a mount
|
||||
# over a file that does not exist yet.
|
||||
systemd.services."container@${cfg.machine}" = caTrust.containerOrdering;
|
||||
|
||||
containers.${cfg.machine} = {
|
||||
autoStart = true;
|
||||
ephemeral = false;
|
||||
|
|
@ -1448,7 +1647,11 @@ in
|
|||
hostPath = bootstrapTokenDir;
|
||||
isReadOnly = true;
|
||||
};
|
||||
};
|
||||
}
|
||||
# The hive CA, for the forwarder's two outbound calls: the swarm
|
||||
# collector's gateway name and authelia's token endpoint. Empty when
|
||||
# the gateway is not self-signed, so the whole path drops out.
|
||||
// caTrust.bindMount;
|
||||
|
||||
# The seal talks to the TPM through the kernel's resource manager, so
|
||||
# the device has to cross the container boundary or the store cannot
|
||||
|
|
@ -1471,7 +1674,27 @@ in
|
|||
imports = [
|
||||
(import ./swarm-container-resolver.nix {
|
||||
inherit (networkCfg) bridgeIp;
|
||||
dnsConsumers = [ "openbao.service" ];
|
||||
# The forwarder resolves two swarm names of its own — the
|
||||
# collector it exports to and the identity provider it mints
|
||||
# a token at — so it belongs in this list for the reason the
|
||||
# module states: an unordered resolver write is a cold-boot
|
||||
# race that shows up arbitrarily far from its cause.
|
||||
dnsConsumers = [
|
||||
"openbao.service"
|
||||
"opentelemetry-collector.service"
|
||||
];
|
||||
})
|
||||
]
|
||||
# `SSL_CERT_FILE` REPLACES the trust store rather than adding to
|
||||
# it, so a failed assembly yields an empty pool and every TLS call
|
||||
# fails while the unit looks healthy. That is why this is the
|
||||
# shared helper — it carries the `Requires` and the non-empty
|
||||
# check — and not a local `cat`.
|
||||
++ [
|
||||
(caTrust.trustBundle {
|
||||
inherit pkgs;
|
||||
name = cfg.machine;
|
||||
consumers = [ "opentelemetry-collector" ];
|
||||
})
|
||||
];
|
||||
|
||||
|
|
@ -1708,6 +1931,34 @@ in
|
|||
storage = "file_storage";
|
||||
};
|
||||
|
||||
# The scrape that used to belong to the swarm collector, now
|
||||
# one hop shorter and one host less fussy — mara:
|
||||
# *"move the existing scraper to the local collector"*.
|
||||
#
|
||||
# ⚠️ `metrics_path` and `params` are STATED, and both have to
|
||||
# be: openbao serves no `/metrics` at all, and
|
||||
# `/v1/sys/metrics` answers JSON unless the format is asked
|
||||
# for. A scrape of the default path 404s and a scrape without
|
||||
# the query parses as nothing — each reads as a dead exporter
|
||||
# rather than as a wrong request, which is how the previous
|
||||
# owner of this scrape learned to spell it
|
||||
# (./swarm-otel.nix's `loopbackScrapeConfig`, still the
|
||||
# renderer for every other service's target).
|
||||
#
|
||||
# No credential, unlike the swarm collector's
|
||||
# `publishedScrapeTargets` jobs: the target is the loopback
|
||||
# listener above, inside this container's own netns, which is
|
||||
# what lets the scrape stay unauthenticated without the API
|
||||
# listener's client-cert requirement being touched.
|
||||
receivers.prometheus.config.scrape_configs = [
|
||||
{
|
||||
job_name = "bao";
|
||||
static_configs = [ { targets = [ metricsScrapeTarget ]; } ];
|
||||
metrics_path = "/v1/sys/metrics";
|
||||
params.format = [ "prometheus" ];
|
||||
}
|
||||
];
|
||||
|
||||
# Identity the hop above cannot supply: a host-side reader can
|
||||
# say which machine a line came from, and only a collector
|
||||
# inside this container can say it was the secret store's. No
|
||||
|
|
@ -1728,7 +1979,34 @@ in
|
|||
# protocol is, which is the same reason
|
||||
# ./hive-c0re/environment.nix pins the protocol it hands out.
|
||||
# `endpoint` is a BASE the exporter appends `/v1/logs` to.
|
||||
exporters.otlphttp.endpoint = otelFirstHop;
|
||||
exporters.otlphttp = {
|
||||
endpoint = otelFirstHop;
|
||||
# The receiver at that address accepts an OIDC token and
|
||||
# nothing else (./swarm-otel.nix's `oidc/*` authenticators),
|
||||
# so this is what makes the hop deliver rather than collect
|
||||
# 401s — unconditional, because a forwarder that exports
|
||||
# unauthenticated has no working mode to fall back to.
|
||||
auth.authenticator = forwarderAuthName;
|
||||
};
|
||||
|
||||
# The push side, opposite in direction to the receiver's
|
||||
# `oidc/*`: those VALIDATE a token arriving, this OBTAINS one
|
||||
# to send. Hence `oauth2client`, and hence a client id and
|
||||
# secret rather than an issuer and an audience to check.
|
||||
extensions.${forwarderAuthName} = {
|
||||
client_id = cfg.otel.clientId;
|
||||
# A path, never a value — and one systemd resolves at
|
||||
# runtime, so neither the secret nor the credentials
|
||||
# directory is rendered into the nix store.
|
||||
client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${forwarderCredentialId}";
|
||||
token_url = "${toString autheliaCfg.url}/api/oidc/token";
|
||||
# ⚠️ THE AUDIENCE HAS TO BE REQUESTED, not merely granted.
|
||||
# Registering it on the client only makes it permissible; a
|
||||
# token minted without asking carries `aud: []` and the
|
||||
# receiver refuses it — with a config that reads perfectly
|
||||
# at both ends.
|
||||
endpoint_params.audience = cfg.otel.clientId;
|
||||
};
|
||||
|
||||
service.pipelines.logs = {
|
||||
receivers = [ "journald" ];
|
||||
|
|
@ -1736,12 +2014,53 @@ in
|
|||
exporters = [ "otlphttp" ];
|
||||
};
|
||||
|
||||
# A SECOND pipeline, not a second receiver on the logs one: a
|
||||
# pipeline is typed, and a prometheus receiver in `logs` is a
|
||||
# config the collector rejects at startup rather than one that
|
||||
# quietly ships metrics as log records.
|
||||
#
|
||||
# Same `resource` processor and same `otlphttp` exporter, so
|
||||
# the store's metrics carry the same `service.name` its logs
|
||||
# do and leave by the same hop — no otel bypass, one exit from
|
||||
# this container. The exporter treats `endpoint` as a base and
|
||||
# appends the signal's own path, so `/v1/metrics` here and
|
||||
# `/v1/logs` above come from the one address.
|
||||
service.pipelines.metrics = {
|
||||
receivers = [ "prometheus" ];
|
||||
processors = [ "resource" ];
|
||||
exporters = [ "otlphttp" ];
|
||||
};
|
||||
|
||||
# An extension configured but not listed here is INERT — the
|
||||
# receiver's `storage: file_storage` above would name a
|
||||
# component the collector never starts.
|
||||
service.extensions = [ "file_storage" ];
|
||||
# component the collector never starts, and the exporter's
|
||||
# authenticator would push unauthenticated while the
|
||||
# collector reports healthy.
|
||||
service.extensions = [
|
||||
"file_storage"
|
||||
forwarderAuthName
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
# How the secret crosses the sandbox. The host unit
|
||||
# `swarm-bao-forwarder-oidc` leaves a root-owned 0400 file at this
|
||||
# path in the container's tree; systemd reads it as root before
|
||||
# the sandbox exists and re-exposes it to whichever uid
|
||||
# `DynamicUser` picked, under the directory the extension above
|
||||
# names through `${env:CREDENTIALS_DIRECTORY}`.
|
||||
#
|
||||
# ⚠️ Unconditional, and a missing source is a unit that refuses to
|
||||
# start. That is the intended shape rather than an oversight: this
|
||||
# forwarder has one way to deliver, so a boot where the secret has
|
||||
# not arrived yet is a collector that says so and retries, not one
|
||||
# that quietly exports into a 401. The restart is what closes the
|
||||
# first-boot window the host unit's own comment describes.
|
||||
systemd.services.opentelemetry-collector.serviceConfig = {
|
||||
LoadCredential = [ "${forwarderCredentialId}:${forwarderSecretInContainer}" ];
|
||||
Restart = "on-failure";
|
||||
RestartSec = 15;
|
||||
};
|
||||
};
|
||||
};
|
||||
})
|
||||
|
|
|
|||
|
|
@ -47,6 +47,24 @@ let
|
|||
# bound once here (rather than copy-pasted at both sites) is the only way
|
||||
# the two can't drift apart.
|
||||
swarmTierName = cfg.producerName;
|
||||
|
||||
# The SECOND swarm-tier producer: the collector inside the secret store's
|
||||
# container (`swarm-bao.nix`). It gets an owner of its own rather than
|
||||
# pushing into `swarmTierName`'s route, because the audience that route
|
||||
# checks is `swarm-controller`'s own client id — one identity per principal,
|
||||
# so a second principal brings its own client, its own audience and
|
||||
# therefore its own authenticator and receiver.
|
||||
#
|
||||
# Named after the store's container, so `otlp/swarm-bao` reads as the thing
|
||||
# that pushes into it. No `reserved-names.nix` entry is needed for it the
|
||||
# way `swarm` has one: the name CONTAINS `swarm`, which
|
||||
# `nix/reserved-hive-fragments.nix` forbids as a substring of any hive name,
|
||||
# so no hive can ever own these components. The assertion below checks that
|
||||
# fragment is still listed rather than trusting it.
|
||||
storeProducerName = baoCfg.machine;
|
||||
storeClientId = baoCfg.otel.clientId;
|
||||
reservedHiveFragments = import ../reserved-hive-fragments.nix;
|
||||
|
||||
# Every `<owner>` no hive may take. Read from `nix/reserved-names.nix`, the
|
||||
# same file the daemons are handed as `HIVE_RESERVED_NAMES`, because agent
|
||||
# names and hive names are ONE namespace going forward — a locally-owned
|
||||
|
|
@ -363,6 +381,50 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
storeProducerPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 4391;
|
||||
description = ''
|
||||
Port the receiver for the **secret store's own forwarder** listens
|
||||
on, at `127.0.0.1`. Reached through the gateway at
|
||||
`https://''${domain}/''${storeProducerName}/`, the same
|
||||
path-per-producer shape a hive and the swarm-tier producer above
|
||||
both use.
|
||||
|
||||
Its own receiver, and not a second sender into
|
||||
{option}`services.hyperhive.swarm.otel.producerPort`, because an
|
||||
`oidc` authenticator checks exactly ONE audience: that receiver
|
||||
admits the audience `swarm-controller` registers for itself, and
|
||||
the store's forwarder is a different principal with a client of its
|
||||
own. Sharing the route would mean sharing that client — which is
|
||||
the thing the store exists to make unnecessary.
|
||||
|
||||
⚠️ A reserved value rather than an offset from
|
||||
{option}`services.hyperhive.swarm.otel.port`, for the reason stated
|
||||
on `producerPort`: that range grows with the hive count and would
|
||||
eventually walk into any fixed offset from it. The collision
|
||||
assertion below covers this port too.
|
||||
'';
|
||||
};
|
||||
|
||||
storeProducerName = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
readOnly = true;
|
||||
default = baoCfg.machine;
|
||||
defaultText = lib.literalExpression "services.hyperhive.swarm.bao.machine";
|
||||
description = ''
|
||||
Path and component name of the store forwarder's own receiver
|
||||
(`otlp/''${storeProducerName}`, `oidc/''${storeProducerName}`).
|
||||
Published so `swarm-bao.nix` addresses the receiver by name rather
|
||||
than repeating the string, exactly as `producerName` above is
|
||||
published for `swarm-controller.nix`.
|
||||
|
||||
Read-only and derived from the store's container name: two
|
||||
spellings of a name both ends have to agree on is a 404 on a
|
||||
request that authenticated perfectly.
|
||||
'';
|
||||
};
|
||||
|
||||
producerName = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
readOnly = true;
|
||||
|
|
@ -619,6 +681,9 @@ in
|
|||
# request reaches the receiver, which knows nothing about the
|
||||
# path it was found at.
|
||||
"/${swarmTierName}/".proxyPass = "http://127.0.0.1:${toString cfg.producerPort}/";
|
||||
# The secret store's forwarder, on its own route for its own
|
||||
# audience — same trailing slashes, same reason.
|
||||
"/${storeProducerName}/".proxyPass = "http://127.0.0.1:${toString cfg.storeProducerPort}/";
|
||||
# There is no swarm-wide inbox, and a closed door is the honest
|
||||
# description of that. Every other route into this collector
|
||||
# belongs to exactly one hive or the swarm-tier producer above.
|
||||
|
|
@ -873,6 +938,25 @@ in
|
|||
Put it back, or give this module a different swarmTierName.
|
||||
'';
|
||||
}
|
||||
{
|
||||
# The store forwarder's components are named after a container whose
|
||||
# name merely CONTAINS the reserved word, so what protects them is
|
||||
# the substring list rather than the equality one. Same reason the
|
||||
# assertion above exists: the list lives in another file, and a guard
|
||||
# whose subject can be edited elsewhere has to assert its own case is
|
||||
# still covered.
|
||||
assertion = lib.any (f: lib.hasInfix f storeProducerName) reservedHiveFragments;
|
||||
message = ''
|
||||
nix/reserved-hive-fragments.nix forbids no substring of
|
||||
'${storeProducerName}', which the swarm collector needs kept out of
|
||||
the hive namespace: it names components `<kind>/<owner>`, so a hive
|
||||
called '${storeProducerName}' would replace the secret store
|
||||
forwarder's own receiver and pipeline entry.
|
||||
|
||||
Put the fragment back, or give the store's container a name one of
|
||||
them covers.
|
||||
'';
|
||||
}
|
||||
{
|
||||
# A published target that is not an `https://host/path` url. Without
|
||||
# this the split returns null and the failure surfaces as
|
||||
|
|
@ -974,6 +1058,7 @@ in
|
|||
others = [
|
||||
cfg.telemetryPort
|
||||
cfg.producerPort
|
||||
cfg.storeProducerPort
|
||||
otelCfg.collector.port
|
||||
]
|
||||
++ lib.optional deployCfg.victoriametrics.enable vmCfg.port;
|
||||
|
|
@ -1119,6 +1204,15 @@ in
|
|||
auth.authenticator = "oidc/${swarmTierName}";
|
||||
};
|
||||
}
|
||||
# The secret store's forwarder, which is a swarm-tier
|
||||
# producer with a principal of its own — a receiver per
|
||||
# audience, because an `oidc` authenticator checks one.
|
||||
// {
|
||||
"otlp/${storeProducerName}".protocols.http = {
|
||||
endpoint = "127.0.0.1:${toString cfg.storeProducerPort}";
|
||||
auth.authenticator = "oidc/${storeProducerName}";
|
||||
};
|
||||
}
|
||||
# MERGED with the per-hive receivers, never assigned over
|
||||
# them. A plain assignment here would drop every hive's
|
||||
# receiver and still render a valid config that starts
|
||||
|
|
@ -1303,7 +1397,13 @@ in
|
|||
# same reasoning as its receiver above: `otlp/${swarmTierName}`
|
||||
# always exists, so the authenticator it names must too, or an
|
||||
# extension-not-listed startup failure follows every deploy.
|
||||
++ [ "oidc/${swarmTierName}" ]
|
||||
++ [
|
||||
"oidc/${swarmTierName}"
|
||||
# Unconditional for the same reason, and one level sharper:
|
||||
# the store runs in every swarm, so its forwarder's
|
||||
# receiver is as permanent as the swarm tier's own.
|
||||
"oidc/${storeProducerName}"
|
||||
]
|
||||
# The push side's authenticators. Same rule as above: one an
|
||||
# exporter names but this list omits is INERT — the collector
|
||||
# starts clean and pushes unauthenticated.
|
||||
|
|
@ -1347,6 +1447,16 @@ in
|
|||
"metrics/${swarmTierName}" = {
|
||||
receivers = [
|
||||
"otlp/${swarmTierName}"
|
||||
# The store forwarder's receiver joins the swarm
|
||||
# tier's pipeline rather than bringing one of its own:
|
||||
# the stamp it needs is `resource/${swarmTierName}`'s
|
||||
# exact stamp — the swarm's name and deliberately no
|
||||
# `hive` — and a second pipeline would be that same
|
||||
# processor under a second name to keep in step. What
|
||||
# a producer is separated FOR is the credential it
|
||||
# proves, and that is the receiver's job, not the
|
||||
# pipeline's.
|
||||
"otlp/${storeProducerName}"
|
||||
]
|
||||
++ lib.optional (cfg.scrapeTargets != { } || cfg.publishedScrapeTargets != { }) "prometheus";
|
||||
processors = [ "resource/${swarmTierName}" ];
|
||||
|
|
@ -1386,7 +1496,14 @@ in
|
|||
# `hive` value to put on them.
|
||||
// lib.optionalAttrs collectLogs {
|
||||
"logs/${swarmTierName}" = {
|
||||
receivers = [ "journald" ];
|
||||
# This host's own journal, and the store container's,
|
||||
# which arrives over the receiver above rather than off
|
||||
# a disk — the store's forwarder reads a journal no
|
||||
# reader on another host can see.
|
||||
receivers = [
|
||||
"journald"
|
||||
"otlp/${storeProducerName}"
|
||||
];
|
||||
processors = [ "resource/${swarmTierName}" ];
|
||||
exporters = logExporterNames;
|
||||
};
|
||||
|
|
@ -1450,6 +1567,18 @@ in
|
|||
audience = config.services.hyperhive.swarm.controller.queueClientId;
|
||||
};
|
||||
}
|
||||
# The secret store forwarder's own authenticator. Same shape
|
||||
# again, and the audience is that principal's own client id
|
||||
# — the self-referential form `swarm-authelia.nix` registers
|
||||
# a hive client under and `swarm-controller` uses above,
|
||||
# read from the option `glue-swarm-bao-otel-oidc-client.nix`
|
||||
# registers so the two ends cannot spell it differently.
|
||||
// {
|
||||
"oidc/${storeProducerName}" = {
|
||||
issuer_url = hyperhiveCfg.swarm.authelia.url;
|
||||
audience = storeClientId;
|
||||
};
|
||||
}
|
||||
# The push side. Opposite direction to every `oidc/*` above —
|
||||
# those VALIDATE a token arriving; these OBTAIN one to send.
|
||||
# Hence `oauth2client` rather than `oidc`, and hence a client
|
||||
|
|
|
|||
|
|
@ -71,6 +71,13 @@ let
|
|||
serviceClientIds = [
|
||||
hyperhiveCfg.swarm.grafana.oidc.clientId
|
||||
hyperhiveCfg.swarm.otel.clientId
|
||||
# The secret store's own journal forwarder. A third swarm service by this
|
||||
# list's own rule — a client registered once for the whole swarm, whose
|
||||
# secret one host mints and another reads — even though the principal
|
||||
# behind it runs inside the store's container rather than beside it.
|
||||
# Missing from here, authelia mints the value and nothing carries it, so
|
||||
# the reader waits on a path that is never written.
|
||||
hyperhiveCfg.swarm.bao.otel.clientId
|
||||
];
|
||||
|
||||
# Where this unit keeps the appservice tokens it minted, and the whole reason
|
||||
|
|
|
|||
Loading…
Reference in a new issue