swarm-nats: forward the queue container's journal to the swarm collector
The swarm-nats container now runs its own otel collector, which reads the
container's journal and pushes it to the swarm collector at
https://<swarm.otel.domain>/swarm-nats, authenticated as the queue's
existing OIDC client (swarm.nats.clientId). Records carry
service.name=nats and no hive label.
The collector lives in a reusable container module,
nix/container-modules/swarm-journal-forwarder.nix, which follows
swarm-bao's forwarder: a journald receiver over /var/log/journal with
merge and a file_storage cursor, an otlphttp exporter with an
oauth2client authenticator that requests its own client id as audience,
self-metrics on a loopback telemetry port, and the client secret handed
in through LoadCredential. Unlike swarm-bao, the secret is fetched inside
the container by swarm-journal-forwarder-oidc.service, under the
container's own store identity (the swarm-nats-auth leaf, which already
holds a read grant on secret/swarm/services/swarm-nats/oidc/client).
The fetch retries until the store answers, and the collector restarts
until the file exists. A 10-minute timer re-reads the secret and restarts
the collector when it changed.
The swarm collector's authenticated producer route was hard-wired to
swarm-bao. It is now rendered from swarm.otel.serviceProducers
(internal), one receiver, oidc authenticator, gateway location and
pipeline entry per service: swarm-bao (port 4391, unchanged names) and
swarm-nats (port 4392). The reserved-fragment and port-collision
assertions cover every entry.
The queue's OIDC client registration gains its own id as audience and
RS256-signed access tokens, which the receiver verifies offline.
swarm-nats drops --link-journal=host, so its journal stays in the guest
where the forwarder reads it, as swarm-bao did in 44009dc51. The new
swarm.nats.otel.telemetryPort (8891) keeps the forwarder off the
collectors' 8888-8890 in the shared host netns, with an assertion.
module-eval-nats-authelia gains four cases: the container's forwarder
and fetch unit on the store host and on a queue-only host, the
responder role's read grant, the swarm collector's swarm-nats receiver
on a host without the queue, and the client registration.
Part of #4499
Part of #5342
This commit is contained in:
atlas
mara
parent
b66a09f65b
commit
40a2bd047b
9 changed files with 660 additions and 105 deletions
315
nix/container-modules/swarm-journal-forwarder.nix
Normal file
315
nix/container-modules/swarm-journal-forwarder.nix
Normal file
|
|
@ -0,0 +1,315 @@
|
|||
# A swarm service container's own journal forwarder: an otel collector that
|
||||
# reads this container's journal and pushes it to the swarm collector by its
|
||||
# gateway name, authenticated as the service's own OIDC client.
|
||||
#
|
||||
# Imported inside the container's own `config`. The host module that declares
|
||||
# `containers.<name>` sets the options below: the container's evaluation is
|
||||
# nested inside the host's, and reading the host side back from in here
|
||||
# recurses.
|
||||
#
|
||||
# The client secret is fetched inside the container, under the container's own
|
||||
# store identity, by `swarm-journal-forwarder-oidc`. That needs the container to
|
||||
# reach the store, authelia and the swarm collector by name: the container needs
|
||||
# a network and a resolver.
|
||||
#
|
||||
# ⚠️ The host must NOT start this container with `--link-journal=host`. That
|
||||
# bind-mounts a host directory over the guest's /var/log/journal which journald
|
||||
# does not write into, and the receiver below reads nothing while the collector
|
||||
# stays healthy. nixpkgs' default `--link-journal=try-guest` keeps the journal
|
||||
# in the guest, where this reads it.
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.services.hyperhive.swarmJournalForwarder;
|
||||
|
||||
atomicWriteSecret = import ../host-modules/lib/atomic-write-secret.nix { };
|
||||
refreshConsumer = import ../host-modules/lib/refresh-consumer.nix { };
|
||||
storeRetry = import ../host-modules/lib/store-retry.nix { };
|
||||
storeRefreshTimer = import ../host-modules/lib/store-refresh-timer.nix { };
|
||||
|
||||
fetchUnit = "swarm-journal-forwarder-oidc";
|
||||
# On the container's /run (tmpfs), so the secret never reaches persistent
|
||||
# disk. The file is not created ahead of the fetch: `LoadCredential` refuses a
|
||||
# missing source, so the collector stays down until the store answers and
|
||||
# then resumes from its saved journal cursor.
|
||||
secretDir = "/run/${fetchUnit}";
|
||||
secretFile = "${secretDir}/client-secret";
|
||||
|
||||
# The collector runs under `DynamicUser` and opens `client_secret_file`
|
||||
# itself. `LoadCredential` reads the file as root and re-exposes it under
|
||||
# `${CREDENTIALS_DIRECTORY}/<id>`, whichever uid the unit got.
|
||||
credentialId = "oidc-client-secret";
|
||||
authName = "oauth2client/swarm";
|
||||
in
|
||||
{
|
||||
options.services.hyperhive.swarmJournalForwarder = {
|
||||
enable = lib.mkEnableOption "this container's own journal forwarder to the swarm collector";
|
||||
|
||||
serviceName = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
example = "nats";
|
||||
description = ''
|
||||
`service.name` stamped on every record. The display identity a
|
||||
dashboard selects on, matching the service's scrape job name. No
|
||||
`hive` label: the swarm collector stamps `swarm` from the receiver
|
||||
that accepted the push.
|
||||
'';
|
||||
};
|
||||
|
||||
endpoint = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
example = "https://otel.swarm.example/swarm-nats";
|
||||
description = ''
|
||||
Base URL of this service's own receiver on the swarm collector:
|
||||
`https://''${swarm.otel.domain}/<producer>`, a route
|
||||
{option}`services.hyperhive.swarm.otel.serviceProducers` declares.
|
||||
The exporter appends `/v1/logs`.
|
||||
'';
|
||||
};
|
||||
|
||||
clientId = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = ''
|
||||
OAuth2 client the forwarder mints its token as, and the audience it
|
||||
asks for. Must equal the `clientId` of this service's
|
||||
{option}`services.hyperhive.swarm.otel.serviceProducers` entry, which
|
||||
is the one audience that receiver accepts.
|
||||
'';
|
||||
};
|
||||
|
||||
tokenUrl = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
example = "https://auth.swarm.example/api/oidc/token";
|
||||
description = "Authelia's token endpoint.";
|
||||
};
|
||||
|
||||
telemetryPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
description = ''
|
||||
Loopback port for the collector's own metrics (refused and dropped
|
||||
records, exporter failures). Required because the container may share
|
||||
the host netns, where the binary's default 8888 is the hive tier's.
|
||||
'';
|
||||
};
|
||||
|
||||
store = {
|
||||
package = lib.mkOption {
|
||||
type = lib.types.package;
|
||||
description = "The `bao` CLI the fetch unit logs in and reads with.";
|
||||
};
|
||||
addr = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "The swarm secret store's address, by name.";
|
||||
};
|
||||
secretPath = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
example = "secret/swarm/services/swarm-nats/oidc/client";
|
||||
description = ''
|
||||
KV path of the client secret. The container's store identity must
|
||||
hold a read grant on it.
|
||||
'';
|
||||
};
|
||||
clientCertFile = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = ''
|
||||
The container's store identity, as a path inside the container. A
|
||||
path, never a value.
|
||||
'';
|
||||
};
|
||||
clientKeyFile = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Private key for `store.clientCertFile`. A path, never a value.";
|
||||
};
|
||||
caFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "CA the store's server certificate chains to; null means the system trust store.";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
assertions = [
|
||||
{
|
||||
# With a volatile journal the receiver finds an empty directory and the
|
||||
# collector starts clean and ships nothing.
|
||||
assertion =
|
||||
!(lib.elem config.services.journald.storage [
|
||||
"volatile"
|
||||
"none"
|
||||
]);
|
||||
message = ''
|
||||
This container forwards its own journal to the swarm collector, but
|
||||
services.journald.storage is "${config.services.journald.storage}".
|
||||
The forwarder reads /var/log/journal, which journald only writes when
|
||||
it stores persistently.
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
services.opentelemetry-collector = {
|
||||
enable = true;
|
||||
# `journald` is a contrib receiver.
|
||||
package = pkgs.opentelemetry-collector-contrib;
|
||||
# A parser, not a wiring check: it accepts an extension that is never
|
||||
# started.
|
||||
validateConfigFile = true;
|
||||
settings = {
|
||||
# The receiver's read cursor, so a restart resumes rather than skipping
|
||||
# what was written while it was down. `create_directory`, because
|
||||
# `otelcol validate` runs in the build sandbox before systemd's
|
||||
# `StateDirectory=` exists.
|
||||
extensions.file_storage = {
|
||||
directory = "/var/lib/opentelemetry-collector";
|
||||
create_directory = true;
|
||||
};
|
||||
|
||||
receivers.journald = {
|
||||
# ⚠️ Stated: the receiver's default is the runtime journal under
|
||||
# /run/log/journal, which is empty when journald stores persistently.
|
||||
directory = "/var/log/journal";
|
||||
# Without `--merge`, `journalctl --follow` over a journal with no
|
||||
# entry for the current boot crash-loops the receiver.
|
||||
merge = true;
|
||||
storage = "file_storage";
|
||||
};
|
||||
|
||||
processors.resource.attributes = [
|
||||
{
|
||||
key = "service.name";
|
||||
value = cfg.serviceName;
|
||||
action = "upsert";
|
||||
}
|
||||
];
|
||||
|
||||
exporters.otlphttp = {
|
||||
endpoint = cfg.endpoint;
|
||||
# The receiver accepts an OIDC token and nothing else.
|
||||
auth.authenticator = authName;
|
||||
};
|
||||
|
||||
extensions.${authName} = {
|
||||
client_id = cfg.clientId;
|
||||
# A path systemd resolves at runtime, so neither the secret nor the
|
||||
# credentials directory is rendered into the nix store.
|
||||
client_secret_file = "\${env:CREDENTIALS_DIRECTORY}/${credentialId}";
|
||||
token_url = cfg.tokenUrl;
|
||||
# ⚠️ Requested, not merely registered: a token minted without asking
|
||||
# carries `aud: []` and the receiver refuses it.
|
||||
endpoint_params.audience = cfg.clientId;
|
||||
};
|
||||
|
||||
service.pipelines.logs = {
|
||||
receivers = [ "journald" ];
|
||||
processors = [ "resource" ];
|
||||
exporters = [ "otlphttp" ];
|
||||
};
|
||||
|
||||
# ⚠️ `readers`, not `metrics.address`: this collector version rejects
|
||||
# `address` at startup.
|
||||
service.telemetry.metrics.readers = [
|
||||
{
|
||||
pull.exporter.prometheus = {
|
||||
host = "127.0.0.1";
|
||||
port = cfg.telemetryPort;
|
||||
};
|
||||
}
|
||||
];
|
||||
|
||||
# An extension not listed here is inert: the receiver's storage would
|
||||
# name a component that never starts, and the exporter would push
|
||||
# unauthenticated.
|
||||
service.extensions = [
|
||||
"file_storage"
|
||||
authName
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
# `Restart` is nixpkgs' `"always"`; a second definition conflicts. Only
|
||||
# `RestartSec` is ours: the 100ms default burns the start-rate limit while
|
||||
# the secret is still on its way, leaving the unit failed instead of
|
||||
# retrying.
|
||||
systemd.services.opentelemetry-collector.serviceConfig = {
|
||||
LoadCredential = [ "${credentialId}:${secretFile}" ];
|
||||
RestartSec = 15;
|
||||
};
|
||||
|
||||
# Reads the client secret with the container's own store identity. A store
|
||||
# or authelia that is not up yet is a failed run that retries
|
||||
# (../host-modules/lib/store-retry.nix), and a collector that restarts until the file lands.
|
||||
systemd.services.${fetchUnit} = {
|
||||
description = "fetch this container's journal forwarder OIDC client secret from the swarm secret store";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
before = [ "opentelemetry-collector.service" ];
|
||||
path = [
|
||||
cfg.store.package
|
||||
pkgs.coreutils
|
||||
pkgs.systemd
|
||||
];
|
||||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||||
serviceConfig = storeRetry.serviceConfig // {
|
||||
# No `RemainAfterExit`: the timer below starts this again, and starting
|
||||
# an active unit is a no-op.
|
||||
Type = "oneshot";
|
||||
SyslogIdentifier = fetchUnit;
|
||||
TimeoutStartSec = 30;
|
||||
};
|
||||
environment = {
|
||||
BAO_ADDR = cfg.store.addr;
|
||||
BAO_CLIENT_CERT = cfg.store.clientCertFile;
|
||||
BAO_CLIENT_KEY = cfg.store.clientKeyFile;
|
||||
}
|
||||
// lib.optionalAttrs (cfg.store.caFile != null) {
|
||||
BAO_CACERT = cfg.store.caFile;
|
||||
};
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
${atomicWriteSecret}
|
||||
${refreshConsumer}
|
||||
|
||||
err="$(mktemp)"
|
||||
trap 'rm -f "$err"' EXIT
|
||||
|
||||
# `-token-only` answers on stdout and skips the token helper, which
|
||||
# needs a `sh` this unit's `path` does not carry.
|
||||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||||
echo "could not log in to the swarm secret store with this container's certificate." >&2
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
export BAO_TOKEN
|
||||
|
||||
if ! secret="$(bao kv get -field=value ${lib.escapeShellArg cfg.store.secretPath} 2>"$err")"; then
|
||||
echo "the store did not return ${cfg.store.secretPath}: the forwarder has no client secret yet." >&2
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$secret" ]; then
|
||||
echo "the store returned an empty ${cfg.store.secretPath}." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# `printf` inside `atomic_write_secret` is bash's own, so the value
|
||||
# never becomes an argument in /proc.
|
||||
install -d -m 0700 ${lib.escapeShellArg secretDir}
|
||||
if secret_differs ${lib.escapeShellArg secretFile} "$secret"; then
|
||||
atomic_write_secret 0400 root:root ${lib.escapeShellArg secretFile} "$secret"
|
||||
fi
|
||||
|
||||
# `LoadCredential` copies the file at start only: a rotated secret
|
||||
# reaches the collector by restarting it.
|
||||
refresh_unit opentelemetry-collector.service ${lib.escapeShellArg secretFile}
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.timers.${fetchUnit} =
|
||||
storeRefreshTimer "re-read this container's journal forwarder OIDC client secret from the swarm secret store";
|
||||
};
|
||||
}
|
||||
|
|
@ -39,6 +39,14 @@ in
|
|||
# responder calls, works either way, so the token endpoint's
|
||||
# `unauthorized_client` is the only place a wrong kind shows.
|
||||
kind = "machine";
|
||||
# The queue container's journal forwarder mints its token as this
|
||||
# client, for the swarm collector's `oidc/swarm-nats` receiver, which
|
||||
# accepts this audience alone (`serviceProducers` in
|
||||
# ./swarm-otel-service.nix).
|
||||
audience = [ natsCfg.clientId ];
|
||||
# That receiver verifies the token offline against `/jwks.json`;
|
||||
# authelia's default opaque handle fails there.
|
||||
accessTokenSignedResponseAlg = "RS256";
|
||||
}
|
||||
];
|
||||
};
|
||||
|
|
|
|||
|
|
@ -107,6 +107,24 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
otel.telemetryPort = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 8891;
|
||||
description = ''
|
||||
Loopback port the journal forwarder inside the queue's container
|
||||
serves its own metrics on (refused and dropped records, exporter
|
||||
failures).
|
||||
|
||||
⚠️ Not 8888, 8889 or 8890: the hive collector, the swarm collector
|
||||
and the store's forwarder bind those
|
||||
({option}`services.hyperhive.otel.telemetryPort`,
|
||||
{option}`services.hyperhive.swarm.otel.telemetryPort`,
|
||||
{option}`services.hyperhive.swarm.bao.otel.telemetryPort`), and this
|
||||
container shares the host netns with them when co-located. The second
|
||||
collector to bind a port dies with `bind: address already in use`.
|
||||
'';
|
||||
};
|
||||
|
||||
clientId = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "swarm-nats";
|
||||
|
|
|
|||
|
|
@ -25,6 +25,16 @@ let
|
|||
|
||||
machine = "swarm-nats";
|
||||
|
||||
# The container's journal forwarder pushes to its own receiver on the swarm
|
||||
# collector, the route `serviceProducers.swarm-nats` in
|
||||
# ./swarm-otel-service.nix declares, by the collector's gateway name. The
|
||||
# port suffix is elided on 443, as in ./swarm-bao.nix.
|
||||
swarmOtelCfg = config.services.hyperhive.swarm.otel;
|
||||
gatewayPortSuffix = lib.optionalString (
|
||||
config.services.hyperhive.gateway.httpsPort != 443
|
||||
) ":${toString config.services.hyperhive.gateway.httpsPort}";
|
||||
forwarderEndpoint = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${machine}";
|
||||
|
||||
tlsCfg = config.services.hyperhive.deploy.hive-controller.tls;
|
||||
gatewayCfg = config.services.hyperhive.gateway;
|
||||
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
||||
|
|
@ -38,7 +48,12 @@ let
|
|||
caBundleModule = caTrust.trustBundle {
|
||||
inherit pkgs;
|
||||
name = machine;
|
||||
consumers = [ "swarm-nats-auth" ];
|
||||
consumers = [
|
||||
"swarm-nats-auth"
|
||||
# Go: `SSL_CERT_FILE` replaces its trust store, for the swarm
|
||||
# collector's and authelia's names.
|
||||
"opentelemetry-collector"
|
||||
];
|
||||
};
|
||||
# Where the responder's credentials live *inside* the container, and the
|
||||
# host path that resolves to. Two names for one location, because the
|
||||
|
|
@ -575,6 +590,40 @@ in
|
|||
both options.
|
||||
'';
|
||||
}
|
||||
{
|
||||
# The forwarder's route and the receiver behind it are two
|
||||
# declarations of one fact; a mismatch is a 404 or a 401 on an export
|
||||
# that looks healthy.
|
||||
assertion = (swarmOtelCfg.serviceProducers.${machine}.clientId or null) == cfg.clientId;
|
||||
message = ''
|
||||
services.hyperhive.swarm.otel.serviceProducers has no entry
|
||||
"${machine}" with clientId "${cfg.clientId}", so the swarm
|
||||
collector has no receiver for the queue container's journal
|
||||
forwarder. Declare it in swarm-otel-service.nix.
|
||||
'';
|
||||
}
|
||||
{
|
||||
assertion =
|
||||
let
|
||||
ports = [
|
||||
cfg.port
|
||||
cfg.monitorPort
|
||||
cfg.metricsPort
|
||||
cfg.otel.telemetryPort
|
||||
config.services.hyperhive.otel.telemetryPort
|
||||
swarmOtelCfg.telemetryPort
|
||||
config.services.hyperhive.swarm.bao.otel.telemetryPort
|
||||
];
|
||||
in
|
||||
lib.length (lib.unique ports) == lib.length ports;
|
||||
message = ''
|
||||
services.hyperhive.swarm.nats.otel.telemetryPort
|
||||
(${toString cfg.otel.telemetryPort}) collides with another port in
|
||||
the host netns: the queue's own ports, or the telemetry port of the
|
||||
hive collector, the swarm collector or the store's forwarder. Pick
|
||||
a free port.
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
# Open the client port on the bridge, so agent containers can reach
|
||||
|
|
@ -601,9 +650,8 @@ in
|
|||
containers.swarm-nats = {
|
||||
autoStart = true;
|
||||
ephemeral = false;
|
||||
# Journal files on the host, not inside the container: nixpkgs hardcodes
|
||||
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
|
||||
extraFlags = [ "--link-journal=host" ];
|
||||
# No `--link-journal=host`: the journal stays in the guest, where the
|
||||
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
|
||||
inherit privateNetwork;
|
||||
# The public trust bundle (empty when the gateway is not self-signed)
|
||||
# and the queue's own TLS leaf, both read-only.
|
||||
|
|
@ -618,19 +666,49 @@ in
|
|||
{
|
||||
imports = [
|
||||
../container-modules/swarm-container.nix
|
||||
../container-modules/swarm-journal-forwarder.nix
|
||||
(import ../container-modules/swarm-container-resolver.nix {
|
||||
inherit (networkCfg) bridgeIp;
|
||||
# The responder introspects authelia BY NAME on every auth
|
||||
# request, and a responder that cannot resolve it denies
|
||||
# every client — so it must not start before the resolver
|
||||
# file exists.
|
||||
dnsConsumers = [ "swarm-nats-auth.service" ];
|
||||
#
|
||||
# The forwarder and its secret fetch dial the swarm collector,
|
||||
# authelia and the store by name too.
|
||||
dnsConsumers = [
|
||||
"swarm-nats-auth.service"
|
||||
"opentelemetry-collector.service"
|
||||
"swarm-journal-forwarder-oidc.service"
|
||||
];
|
||||
})
|
||||
caBundleModule
|
||||
];
|
||||
|
||||
services.hyperhive.swarmContainer = { inherit privateNetwork; };
|
||||
|
||||
# Authenticated as the queue's own OIDC client, whose secret the
|
||||
# responder's store identity already reads: one identity for the
|
||||
# whole container.
|
||||
services.hyperhive.swarmJournalForwarder = {
|
||||
enable = true;
|
||||
# The scrape job's name, so the queue's logs and metrics carry one
|
||||
# `service.name`.
|
||||
serviceName = "nats";
|
||||
endpoint = forwarderEndpoint;
|
||||
inherit (cfg) clientId;
|
||||
tokenUrl = "${toString autheliaUrl}/api/oidc/token";
|
||||
telemetryPort = cfg.otel.telemetryPort;
|
||||
store = {
|
||||
package = baoDeploy.package;
|
||||
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
secretPath = "secret/swarm/services/${cfg.clientId}/oidc/client";
|
||||
clientCertFile = inContainer "bao-client.pem";
|
||||
clientKeyFile = inContainer "bao-client-key.pem";
|
||||
caFile = if authStoreFiles ? "bao-ca.pem" then inContainer "bao-ca.pem" else null;
|
||||
};
|
||||
};
|
||||
|
||||
services.nats = {
|
||||
enable = true;
|
||||
|
||||
|
|
|
|||
|
|
@ -166,6 +166,48 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
serviceProducers = lib.mkOption {
|
||||
internal = true;
|
||||
type = lib.types.attrsOf (
|
||||
lib.types.submodule {
|
||||
options = {
|
||||
clientId = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = ''
|
||||
The producer's OAuth2 client id: the one audience its
|
||||
receiver's `oidc/<name>` authenticator accepts, and the one
|
||||
the forwarder requests.
|
||||
'';
|
||||
};
|
||||
port = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
description = ''
|
||||
Loopback port of the producer's receiver on the collector's
|
||||
host. In the port-collision assertion with every other
|
||||
receiver there.
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
||||
);
|
||||
default = { };
|
||||
description = ''
|
||||
Swarm services whose own container forwards to this collector, keyed
|
||||
by the route name: each gets a receiver `otlp/<name>`, an
|
||||
authenticator `oidc/<name>` and a gateway location
|
||||
`https://''${domain}/<name>/`, and its records land in the swarm
|
||||
tier's pipelines with no `hive` label.
|
||||
|
||||
A receiver per producer because an `oidc` authenticator checks one
|
||||
audience, and each service mints its token as its own client.
|
||||
|
||||
Declared below for every host, under no `enable`: the collector's
|
||||
host renders the receiver whether or not the service runs there.
|
||||
Every name must contain a fragment of
|
||||
`nix/reserved-hive-fragments.nix`, so no hive can own its components.
|
||||
'';
|
||||
};
|
||||
|
||||
storeProducerName = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
readOnly = true;
|
||||
|
|
@ -346,6 +388,20 @@ in
|
|||
};
|
||||
};
|
||||
|
||||
config.services.hyperhive.swarm.otel.serviceProducers = {
|
||||
# The forwarder in ./swarm-bao.nix.
|
||||
${cfg.storeProducerName} = {
|
||||
clientId = baoCfg.otel.clientId;
|
||||
port = cfg.storeProducerPort;
|
||||
};
|
||||
# The forwarder in ./swarm-nats.nix, which addresses this route by the
|
||||
# queue container's name.
|
||||
swarm-nats = {
|
||||
clientId = hyperhiveCfg.swarm.nats.clientId;
|
||||
port = lib.mkDefault 4392;
|
||||
};
|
||||
};
|
||||
|
||||
# Not under any `enable`: see `publishedScrapeTargets` for why every host
|
||||
# declares every entry.
|
||||
#
|
||||
|
|
|
|||
|
|
@ -41,21 +41,21 @@ let
|
|||
# both sites) is the only way the two can't drift apart.
|
||||
swarmTierName = cfg.producerName;
|
||||
|
||||
# The SECOND swarm-tier producer: the collector inside the secret store's
|
||||
# container (`swarm-bao.nix`). It gets an owner of its own rather than
|
||||
# pushing into `swarmTierName`'s route, because the audience that route
|
||||
# checks is `swarm-controller`'s own client id — one identity per principal,
|
||||
# so a second principal brings its own client, its own audience and
|
||||
# therefore its own authenticator and receiver.
|
||||
# The other swarm-tier producers: the forwarder inside each swarm service's
|
||||
# own container (`serviceProducers` in ./swarm-otel-service.nix). Each gets
|
||||
# an owner of its own rather than pushing into `swarmTierName`'s route,
|
||||
# because the audience that route checks is `swarm-controller`'s own client
|
||||
# id — one identity per principal, so each principal brings its own client,
|
||||
# its own audience and therefore its own authenticator and receiver.
|
||||
#
|
||||
# Named after the store's container, so `otlp/swarm-bao` reads as the thing
|
||||
# that pushes into it. No `reserved-names.nix` entry is needed for it the
|
||||
# way `swarm` has one: the name CONTAINS `swarm`, which
|
||||
# Named after the service's container, so `otlp/swarm-bao` reads as the
|
||||
# thing that pushes into it. No `reserved-names.nix` entry is needed the way
|
||||
# `swarm` has one: each name CONTAINS `swarm`, which
|
||||
# `nix/reserved-hive-fragments.nix` forbids as a substring of any hive name,
|
||||
# so no hive can ever own these components. The assertion below checks that
|
||||
# fragment is still listed rather than trusting it.
|
||||
storeProducerName = baoCfg.machine;
|
||||
storeClientId = baoCfg.otel.clientId;
|
||||
serviceProducers = cfg.serviceProducers;
|
||||
serviceProducerNames = lib.attrNames serviceProducers;
|
||||
reservedHiveFragments = import ../reserved-hive-fragments.nix;
|
||||
|
||||
# Every `<owner>` no hive may take. Read from `nix/reserved-names.nix`, the
|
||||
|
|
@ -206,16 +206,16 @@ let
|
|||
# `--link-journal=try-guest`, which puts the journal inside the container and
|
||||
# leaves the host with a symlink into that container's transient root: a
|
||||
# reader here cannot follow it, and it dangles the moment the container
|
||||
# stops. Every container block except `swarm-bao` therefore sets
|
||||
# stops. Every container block except `swarm-bao` and `swarm-nats` therefore sets
|
||||
# `extraFlags = [ "--link-journal=host" ]`, which the invocation expands
|
||||
# AFTER the hardcoded flag, so the files land here under their own
|
||||
# machine-id subdirectory and are bind-mounted into the guest instead.
|
||||
#
|
||||
# ⚠️ Drop that flag from one of THOSE containers and this receiver silently
|
||||
# stops seeing it — no error, just a unit that never appears in the store.
|
||||
# `swarm-bao` is the one exception on purpose: its journal stays inside the
|
||||
# container for its own collector instead (see `swarm-bao.nix`), so it was
|
||||
# never one of the units this receiver could see either.
|
||||
# `swarm-bao` and `swarm-nats` are the exceptions on purpose: their journals
|
||||
# stay inside the container for its own collector instead (see
|
||||
# `serviceProducers`), so this receiver never sees their units.
|
||||
hostJournalDir = "/var/log/journal";
|
||||
|
||||
# Each store's OTLP route, by domain. One binding because the same string is
|
||||
|
|
@ -380,14 +380,16 @@ in
|
|||
# request reaches the receiver, which knows nothing about the
|
||||
# path it was found at.
|
||||
"/${swarmTierName}/".proxyPass = "http://127.0.0.1:${toString cfg.producerPort}/";
|
||||
# The secret store's forwarder, on its own route for its own
|
||||
# audience — same trailing slashes, same reason.
|
||||
"/${storeProducerName}/".proxyPass = "http://127.0.0.1:${toString cfg.storeProducerPort}/";
|
||||
# There is no swarm-wide inbox, and a closed door is the honest
|
||||
# description of that. Every other route into this collector
|
||||
# belongs to exactly one hive or the swarm-tier producer above.
|
||||
# belongs to exactly one hive or one swarm-tier producer.
|
||||
"/".return = "404";
|
||||
};
|
||||
}
|
||||
# Each service forwarder, on its own route for its own audience —
|
||||
# same trailing slashes, same reason.
|
||||
// lib.mapAttrs' (
|
||||
name: p: lib.nameValuePair "/${name}/" { proxyPass = "http://127.0.0.1:${toString p.port}/"; }
|
||||
) serviceProducers;
|
||||
};
|
||||
|
||||
# The identities this tier authenticates against. Stated rather than
|
||||
|
|
@ -673,21 +675,21 @@ in
|
|||
'';
|
||||
}
|
||||
{
|
||||
# The store forwarder's components are named after a container whose
|
||||
# name merely CONTAINS the reserved word, so what protects them is
|
||||
# the substring list rather than the equality one. Same reason the
|
||||
# The service forwarders' components are named after containers
|
||||
# whose names merely CONTAIN the reserved word, so what protects them
|
||||
# is the substring list rather than the equality one. Same reason the
|
||||
# assertion above exists: the list lives in another file, and a guard
|
||||
# whose subject can be edited elsewhere has to assert its own case is
|
||||
# still covered.
|
||||
assertion = lib.any (f: lib.hasInfix f storeProducerName) reservedHiveFragments;
|
||||
assertion = lib.all (n: lib.any (f: lib.hasInfix f n) reservedHiveFragments) serviceProducerNames;
|
||||
message = ''
|
||||
nix/reserved-hive-fragments.nix forbids no substring of
|
||||
'${storeProducerName}', which the swarm collector needs kept out of
|
||||
the hive namespace: it names components `<kind>/<owner>`, so a hive
|
||||
called '${storeProducerName}' would replace the secret store
|
||||
nix/reserved-hive-fragments.nix forbids no substring of one of
|
||||
${lib.concatStringsSep ", " serviceProducerNames}, which the swarm
|
||||
collector needs kept out of the hive namespace: it names components
|
||||
`<kind>/<owner>`, so a hive with that name would replace a service
|
||||
forwarder's own receiver and pipeline entry.
|
||||
|
||||
Put the fragment back, or give the store's container a name one of
|
||||
Put the fragment back, or give the service's route a name one of
|
||||
them covers.
|
||||
'';
|
||||
}
|
||||
|
|
@ -776,9 +778,9 @@ in
|
|||
others = [
|
||||
cfg.telemetryPort
|
||||
cfg.producerPort
|
||||
cfg.storeProducerPort
|
||||
otelCfg.collector.port
|
||||
];
|
||||
]
|
||||
++ map (p: p.port) (lib.attrValues serviceProducers);
|
||||
all = derived ++ others;
|
||||
in
|
||||
lib.length (lib.unique all) == lib.length all;
|
||||
|
|
@ -922,15 +924,18 @@ in
|
|||
auth.authenticator = "oidc/${swarmTierName}";
|
||||
};
|
||||
}
|
||||
# The secret store's forwarder, which is a swarm-tier
|
||||
# producer with a principal of its own — a receiver per
|
||||
# audience, because an `oidc` authenticator checks one.
|
||||
// {
|
||||
"otlp/${storeProducerName}".protocols.http = {
|
||||
endpoint = "127.0.0.1:${toString cfg.storeProducerPort}";
|
||||
auth.authenticator = "oidc/${storeProducerName}";
|
||||
};
|
||||
}
|
||||
# Each service forwarder, a swarm-tier producer with a
|
||||
# principal of its own — a receiver per audience, because an
|
||||
# `oidc` authenticator checks one.
|
||||
// lib.mapAttrs' (
|
||||
name: p:
|
||||
lib.nameValuePair "otlp/${name}" {
|
||||
protocols.http = {
|
||||
endpoint = "127.0.0.1:${toString p.port}";
|
||||
auth.authenticator = "oidc/${name}";
|
||||
};
|
||||
}
|
||||
) serviceProducers
|
||||
# MERGED with the per-hive receivers, never assigned over
|
||||
# them. A plain assignment here would drop every hive's
|
||||
# receiver and still render a valid config that starts
|
||||
|
|
@ -1128,13 +1133,10 @@ in
|
|||
# same reasoning as its receiver above: `otlp/${swarmTierName}`
|
||||
# always exists, so the authenticator it names must too, or an
|
||||
# extension-not-listed startup failure follows every deploy.
|
||||
++ [
|
||||
"oidc/${swarmTierName}"
|
||||
# Unconditional for the same reason, and one level sharper:
|
||||
# the store runs in every swarm, so its forwarder's
|
||||
# receiver is as permanent as the swarm tier's own.
|
||||
"oidc/${storeProducerName}"
|
||||
]
|
||||
++ [ "oidc/${swarmTierName}" ]
|
||||
# Unconditional for the same reason: a service forwarder's
|
||||
# receiver exists whether or not the service runs on this host.
|
||||
++ map (n: "oidc/${n}") serviceProducerNames
|
||||
# The push side's authenticators. Same rule as above: one an
|
||||
# exporter names but this list omits is INERT — the collector
|
||||
# starts clean and pushes unauthenticated.
|
||||
|
|
@ -1178,17 +1180,16 @@ in
|
|||
"metrics/${swarmTierName}" = {
|
||||
receivers = [
|
||||
"otlp/${swarmTierName}"
|
||||
# The store forwarder's receiver joins the swarm
|
||||
# tier's pipeline rather than bringing one of its own:
|
||||
# the stamp it needs is `resource/${swarmTierName}`'s
|
||||
# exact stamp — the swarm's name and deliberately no
|
||||
# `hive` — and a second pipeline would be that same
|
||||
# processor under a second name to keep in step. What
|
||||
# a producer is separated FOR is the credential it
|
||||
# proves, and that is the receiver's job, not the
|
||||
# pipeline's.
|
||||
"otlp/${storeProducerName}"
|
||||
]
|
||||
# The service forwarders' receivers join the swarm
|
||||
# tier's pipeline rather than bringing their own: the
|
||||
# stamp they need is `resource/${swarmTierName}`'s exact
|
||||
# stamp — the swarm's name and deliberately no `hive` —
|
||||
# and a second pipeline would be that same processor
|
||||
# under a second name to keep in step. What a producer is
|
||||
# separated FOR is the credential it proves, and that is
|
||||
# the receiver's job, not the pipeline's.
|
||||
++ map (n: "otlp/${n}") serviceProducerNames
|
||||
++ lib.optional (cfg.scrapeTargets != { } || cfg.publishedScrapeTargets != { }) "prometheus";
|
||||
processors = [ "resource/${swarmTierName}" ];
|
||||
exporters = exporterNames;
|
||||
|
|
@ -1227,14 +1228,11 @@ in
|
|||
# `hive` value to put on them.
|
||||
// lib.optionalAttrs collectLogs {
|
||||
"logs/${swarmTierName}" = {
|
||||
# This host's own journal, and the store container's,
|
||||
# which arrives over the receiver above rather than off
|
||||
# a disk — the store's forwarder reads a journal no
|
||||
# reader on another host can see.
|
||||
receivers = [
|
||||
"journald"
|
||||
"otlp/${storeProducerName}"
|
||||
];
|
||||
# This host's own journal, and each service container's,
|
||||
# which arrives over its forwarder's receiver above
|
||||
# rather than off a disk — each forwarder reads a journal
|
||||
# no reader on another host can see.
|
||||
receivers = [ "journald" ] ++ map (n: "otlp/${n}") serviceProducerNames;
|
||||
processors = [
|
||||
"filter/exclude-user-sessions"
|
||||
"resource/${swarmTierName}"
|
||||
|
|
@ -1323,18 +1321,19 @@ in
|
|||
audience = config.services.hyperhive.swarm.controller.queueClientId;
|
||||
};
|
||||
}
|
||||
# The secret store forwarder's own authenticator. Same shape
|
||||
# Each service forwarder's own authenticator. Same shape
|
||||
# again, and the audience is that principal's own client id
|
||||
# — the self-referential form `swarm-authelia.nix` registers
|
||||
# a hive client under and `swarm-controller` uses above,
|
||||
# read from the option `glue-swarm-bao-otel-oidc-client.nix`
|
||||
# registers so the two ends cannot spell it differently.
|
||||
// {
|
||||
"oidc/${storeProducerName}" = {
|
||||
# read from the option the client's registration reads so
|
||||
# the two ends cannot spell it differently.
|
||||
// lib.mapAttrs' (
|
||||
name: p:
|
||||
lib.nameValuePair "oidc/${name}" {
|
||||
issuer_url = hyperhiveCfg.swarm.authelia.url;
|
||||
audience = storeClientId;
|
||||
};
|
||||
}
|
||||
audience = p.clientId;
|
||||
}
|
||||
) serviceProducers
|
||||
# The push side. Opposite direction to every `oidc/*` above —
|
||||
# those VALIDATE a token arriving; these OBTAIN one to send.
|
||||
# Hence `oauth2client` rather than `oidc`, and hence a client
|
||||
|
|
|
|||
|
|
@ -560,6 +560,79 @@ let
|
|||
in
|
||||
!(lib.hasInfix "127.0.0.1" url) && !(lib.hasInfix "localhost" url) && !(lib.hasInfix bridgeIp url);
|
||||
}
|
||||
{
|
||||
# The queue container's own journal forwarder, on the store's host and
|
||||
# on a queue host with no store and no swarm collector: it exports to
|
||||
# the swarm collector's `swarm-nats` route by name, authenticated as
|
||||
# the queue's client, and its secret is read with the responder's
|
||||
# store identity inside the container.
|
||||
name = "the queue container forwards its journal to its own route on the swarm collector";
|
||||
ok =
|
||||
lib.all
|
||||
(
|
||||
h:
|
||||
let
|
||||
c = h.containers.swarm-nats.config;
|
||||
s = c.services.opentelemetry-collector.settings;
|
||||
otel = h.services.hyperhive.swarm.otel;
|
||||
fetch = c.systemd.services.swarm-journal-forwarder-oidc;
|
||||
clientId = h.services.hyperhive.swarm.nats.clientId;
|
||||
in
|
||||
c.services.opentelemetry-collector.enable
|
||||
&& s.exporters.otlphttp.endpoint == "https://${otel.domain}/swarm-nats"
|
||||
&& s.exporters.otlphttp.auth.authenticator == "oauth2client/swarm"
|
||||
&& s.extensions."oauth2client/swarm".client_id == clientId
|
||||
&& s.extensions."oauth2client/swarm".endpoint_params.audience == clientId
|
||||
&& lib.elem "oauth2client/swarm" s.service.extensions
|
||||
&& s.service.pipelines.logs.receivers == [ "journald" ]
|
||||
&& lib.elem "oidc-client-secret:/run/swarm-journal-forwarder-oidc/client-secret" c.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
|
||||
&& fetch.environment.BAO_CLIENT_CERT == "/var/lib/swarm-nats-auth/bao-client.pem"
|
||||
&& lib.hasInfix "bao kv get -field=value secret/swarm/services/${clientId}/oidc/client" fetch.script
|
||||
# The guest keeps its journal; a host-linked one is empty inside.
|
||||
&& !(lib.elem "--link-journal=host" h.containers.swarm-nats.extraFlags)
|
||||
)
|
||||
[
|
||||
natsOnStoreHost
|
||||
natsWithStoreIdentity
|
||||
];
|
||||
}
|
||||
{
|
||||
# The read grant the fetch relies on: the responder's role, the identity
|
||||
# the forwarder logs in with, reads the queue client's secret.
|
||||
name = "the queue container's store identity may read the forwarder's client secret";
|
||||
ok = lib.hasInfix "path \"secret/data/swarm/services/swarm-nats/oidc/client\" {" natsOnStoreHost.systemd.services.swarm-bao-nats-auth-policy.script;
|
||||
}
|
||||
{
|
||||
# The far end: a receiver, authenticator, pipeline entry and gateway
|
||||
# route of its own, checking the queue client as audience. Rendered on
|
||||
# a collector host that runs no queue.
|
||||
name = "the swarm collector admits the queue's forwarder on a receiver of its own";
|
||||
ok =
|
||||
let
|
||||
h = shared.baoWithCollector;
|
||||
otel = h.services.hyperhive.swarm.otel;
|
||||
s = h.containers.swarm-otel.config.services.opentelemetry-collector.settings;
|
||||
in
|
||||
!h.services.hyperhive.deploy.nats.enable
|
||||
&& s.receivers."otlp/swarm-nats".protocols.http.auth.authenticator == "oidc/swarm-nats"
|
||||
&& s.extensions."oidc/swarm-nats".audience == h.services.hyperhive.swarm.nats.clientId
|
||||
&& lib.elem "oidc/swarm-nats" s.service.extensions
|
||||
&& lib.elem "otlp/swarm-nats" s.service.pipelines."logs/${otel.producerName}".receivers
|
||||
&& h.services.nginx.virtualHosts.${otel.domain}.locations ? "/swarm-nats/"
|
||||
# The store's forwarder keeps its own receiver beside it.
|
||||
&& s.extensions."oidc/swarm-bao".audience == h.services.hyperhive.swarm.bao.otel.clientId;
|
||||
}
|
||||
{
|
||||
# The token the forwarder mints must name its own client as audience
|
||||
# and be a JWT, or the receiver refuses it.
|
||||
name = "the queue's client is registered with its own audience and a signed access token";
|
||||
ok =
|
||||
let
|
||||
clients = shared.autheliaOnly.services.hyperhive.swarm.authelia.oidc.clients;
|
||||
c = lib.findFirst (c: c.id == "swarm-nats") null clients;
|
||||
in
|
||||
c != null && c.audience == [ "swarm-nats" ] && c.accessTokenSignedResponseAlg == "RS256";
|
||||
}
|
||||
];
|
||||
in
|
||||
runGroup "nats-authelia" cases
|
||||
|
|
|
|||
Loading…
Reference in a new issue