vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-victoriametrics: forward the container's journal to the swarm collector

Browse source 
The metrics store's container now runs the journal forwarder the other
service containers run, pushing to a receiver of its own on the swarm
collector (serviceProducers swarm-victoriametrics) as a new machine OIDC
client (swarm.victoriametrics.otel.clientId), registered by a glue file
wherever authelia runs and published to the store by the secret publisher.

The container keeps its loopback-only netns. The collector cannot dial a
unix socket, so swarm-victoriametrics-egress-relay, a socket-activated
systemd-socket-proxyd inside the container, takes its connections on
127.0.0.1:4380 and passes them to /run/swarm-victoriametrics-egress/egress.sock.
The host's nginx listens there and sends /swarm-victoriametrics/ to the swarm
collector and /api/oidc/token to authelia, both by name and verified, and
refuses every other path. nginx gets ReadWritePaths on that directory and
removes a stale socket before it starts.

The container cannot reach the store, so the client secret is fetched on the
host by swarm-victoriametrics-forwarder-oidc under swarm-metrics-auth's leaf,
the shape swarm-bao-forwarder-oidc has, and bound into the container
read-only. The forwarder module gains hostSecretFile for that: set, it renders
no fetch unit. swarm-metrics-auth's policy gains one read stanza on the
forwarder's client path.

The container drops --link-journal=host, and the host collector stops reading
it.

Also lists the forge publisher's and matrix-ctl's leaves in the forwarder
secrets row of docs/swarm/secrets.md, which the hive-forge/hive-matrix change
missed.
This commit is contained in:
atlas 2026-10-16 12:05:10 +02:00 • committed by mara
commit 7294707d7d
15 changed files with 454 additions and 62 deletions
Download patch file Download diff file Expand all files Collapse all files

31
nix/container-modules/swarm-journal-forwarder.nix
View file

@ -10,7 +10,9 @@
# The client secret is fetched inside the container, under the container's own
# store identity, by `swarm-journal-forwarder-oidc`. That needs the container to
# reach the store, authelia and the swarm collector by name: the container needs
# a network and a resolver.
# a network and a resolver. A container with neither gets the secret from the
# host instead (`hostSecretFile`), and reaches the collector and authelia
# through whatever `endpoint` and `tokenUrl` name.
#
# ⚠️ The host must NOT start this container with `--link-journal=host`. That
# bind-mounts a host directory over the guest's /var/log/journal which journald
@ -38,6 +40,8 @@ let
# then resumes from its saved journal cursor.
secretDir = "/run/${fetchUnit}";
secretFile = "${secretDir}/client-secret";
fetchesHere = cfg.hostSecretFile == null;
credentialSource = if fetchesHere then secretFile else cfg.hostSecretFile;
# The collector runs under `DynamicUser` and opens `client_secret_file`
# itself. `LoadCredential` reads the file as root and re-exposes it under
@ -67,7 +71,8 @@ in
Base URL of this service's own receiver on the swarm collector:
`https://''${swarm.otel.domain}/<producer>`, a route
{option}`services.hyperhive.swarm.otel.serviceProducers` declares.
The exporter appends `/v1/logs`.
The exporter appends `/v1/logs`. In a container with no network, the
same path on a loopback relay that leads there.
'';
};
@ -96,6 +101,19 @@ in
'';
};
hostSecretFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = ''
The client secret as a file the host fetched and bound in, as a path
inside the container. Set, the container fetches nothing and the
`store` options are unused: the shape for a container that cannot
reach the store. The host writes the file only once fetched, and
restarts the collector when it changes. Null, the fetch unit reads it
with `store.*`.
'';
};
store = {
package = lib.mkOption {
type = lib.types.package;
@ -235,14 +253,14 @@ in
# the secret is still on its way, leaving the unit failed instead of
# retrying.
systemd.services.opentelemetry-collector.serviceConfig = {
LoadCredential = [ "${credentialId}:${secretFile}" ];
LoadCredential = [ "${credentialId}:${credentialSource}" ];
RestartSec = 15;
};
# Reads the client secret with the container's own store identity. A store
# or authelia that is not up yet is a failed run that retries
# (../host-modules/lib/store-retry.nix), and a collector that restarts until the file lands.
systemd.services.${fetchUnit} = {
systemd.services.${fetchUnit} = lib.mkIf fetchesHere {
description = "fetch this container's journal forwarder OIDC client secret from the swarm secret store";
wantedBy = [ "multi-user.target" ];
before = [ "opentelemetry-collector.service" ];
@ -309,7 +327,8 @@ in
'';
};
systemd.timers.${fetchUnit} =
storeRefreshTimer "re-read this container's journal forwarder OIDC client secret from the swarm secret store";
systemd.timers.${fetchUnit} = lib.mkIf fetchesHere (
storeRefreshTimer "re-read this container's journal forwarder OIDC client secret from the swarm secret store"
);
};
}

1
nix/host-modules/default.nix
View file

@ -58,6 +58,7 @@
./glue-swarm-bao-otel-oidc-client.nix
./glue-swarm-otel-oidc-client.nix
./glue-victorialogs-otel-oidc-client.nix
./glue-victoriametrics-otel-oidc-client.nix
./swarm-authelia-service.nix
./swarm-authelia.nix
./swarm-bao-service.nix

37
nix/host-modules/glue-victoriametrics-otel-oidc-client.nix Normal file
View file

@ -0,0 +1,37 @@
# Glue: register the metrics store container's journal forwarder as an OIDC
# client wherever authelia runs.
#
# ONE PAIRING PER FILE: that forwarder ← authelia, and nothing else.
# Without it authelia mints no secret, the publisher has nothing to copy into
# the store, and the forwarder's export is a 401.
#
# Gated on authelia being HERE, not on this host running the metrics store: a
# client is a row in THIS host's provider config, the split
# ./glue-swarm-bao-otel-oidc-client.nix makes for the same reason.
{
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
clientId = hyperhiveCfg.swarm.victoriametrics.otel.clientId;
in
{
config = lib.mkIf hyperhiveCfg.deploy.authelia.enable {
services.hyperhive.swarm.authelia.oidc.clients = [
{
id = clientId;
description = "HyperHive metrics store journal forwarder";
kind = "machine";
redirectUris = [ ];
# Its own id as its own permitted audience, the one the swarm
# collector's receiver for this container checks.
audience = [ clientId ];
# The receiver verifies the token offline against `/jwks.json`;
# authelia's default opaque handle fails there.
accessTokenSignedResponseAlg = "RS256";
}
];
};
}

13
nix/host-modules/swarm-bao.nix
View file

@ -1038,12 +1038,16 @@ let
];
# The metrics store's responder, which checks the bearer Grafana presents
# against the one stored for it, and reads nothing else.
# against the one stored for it; and the host unit that fetches the store
# container's journal forwarder secret under the same leaf. Nothing else.
metricsAuthReaders = [
{
name = "swarm-metrics-auth";
cn = baoDeploy.metricsAuthCommonName;
policyText = readStanza "${credentialMountPath}/data/${grafanaMetricsBearerLeaf}";
policyText =
readStanza "${credentialMountPath}/data/${grafanaMetricsBearerLeaf}"
# The metrics store container's journal forwarder, fetched on its host.
+ readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.victoriametrics.otel.clientId}/oidc/client";
}
];
@ -1944,7 +1948,10 @@ in
Subject the store's `swarm-metrics-auth` cert-auth role accepts: the
identity the metrics store's Grafana auth responder presents to read
the bearer Grafana's datasource sends,
`swarm/services/<grafana client id>/metrics/bearer`, and nothing else.
`swarm/services/<grafana client id>/metrics/bearer`, and the one
`swarm-victoriametrics-forwarder-oidc` presents to read
`swarm/services/<metrics store forwarder client id>/oidc/client`.
Nothing else.
⚠️ Reserved as a hive name by ./swarm.nix, like its siblings.
'';

6
nix/host-modules/swarm-otel-service.nix
View file

@ -439,6 +439,12 @@ in
clientId = hyperhiveCfg.swarm.matrix.otel.clientId;
port = lib.mkDefault 4400;
};
# The forwarder in ./swarm-victoriametrics.nix, which reaches this route
# through its host's nginx.
swarm-victoriametrics = {
clientId = hyperhiveCfg.swarm.victoriametrics.otel.clientId;
port = lib.mkDefault 4401;
};
};
# Not under any `enable`: see `publishedScrapeTargets` for why every host

6
nix/host-modules/swarm-otel.nix
View file

@ -216,9 +216,9 @@ let
# The `serviceProducers` containers (`swarm-bao`, `swarm-nats`,
# `swarm-builder`, `swarm-claude-login`, `swarm-review-classifier`,
# `swarm-ci-runner`, `swarm-grafana`, `swarm-victorialogs`, `hive-forge`,
# `hive-matrix`) are the exceptions on purpose: their journals stay inside
# the container for its own collector instead, so this receiver never sees
# their units.
# `hive-matrix`, `swarm-victoriametrics`) are the exceptions on purpose: their
# journals stay inside the container for its own collector instead, so this
# receiver never sees their units.
hostJournalDir = "/var/log/journal";
# Each store's OTLP route, by domain. One binding because the same string is

2
nix/host-modules/swarm-secret-publisher.nix
View file

@ -106,6 +106,8 @@ let
hyperhiveCfg.swarm.victorialogs.otel.clientId
hyperhiveCfg.swarm.forge.otel.clientId
hyperhiveCfg.swarm.matrix.otel.clientId
# The metrics store container's journal forwarder, whose host reads it.
hyperhiveCfg.swarm.victoriametrics.otel.clientId
];
# The swarm controller's OIDC client, which it reads back from the store

16
nix/host-modules/swarm-victoriametrics-service.nix
View file

@ -31,6 +31,22 @@ in
'';
};
otel.clientId = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "swarm-victoriametrics-collector";
description = ''
OAuth2 client id the journal forwarder inside the metrics store's
container authenticates as, and the audience it asks its token for.
Read-only because three hosts have to agree on it: authelia
registers the client (`glue-victoriametrics-otel-oidc-client.nix`),
the swarm collector admits it on the container's receiver
(`swarm.otel.serviceProducers`), and the secret publisher carries
its secret to the store.
'';
};
domain = lib.mkOption {
type = lib.types.str;
default = "metrics.${domainBase}";

231
nix/host-modules/swarm-victoriametrics.nix
View file

@ -22,8 +22,9 @@ let
gatewayCfg = hyperhiveCfg.gateway;
# A netns of its own with no interface but loopback: the store makes no
# outbound calls, and the gateway reaches it over `socketPath` below, so
# nothing on the host or the hive bridge can dial its listener.
# outbound calls, the gateway reaches it over `socketPath` below, and the
# journal forwarder leaves only through `egressSocket`, so nothing on the host
# or the hive bridge can dial its listener.
privateNetwork = true;
# Fixed, not an option: the directory's mode is what keeps the socket
@ -126,6 +127,72 @@ let
# The path Grafana's host creates the bearer at, under the KV mount. The
# same leaf as `grafanaMetricsBearerLeaf` in ./swarm-bao.nix.
grafanaBearerStorePath = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer";
# ── the container's journal forwarder ───────────────────────────────
#
# With no interface but loopback, the forwarder reaches nothing by name. A
# relay in the container listens on `relayPort` and forwards each connection
# to `egressSocket`, a socket this host's nginx listens on and binds into
# the container. nginx sends `/<machine>/` to the swarm collector's route
# (`serviceProducers.swarm-victoriametrics` in ./swarm-otel-service.nix) and
# the token endpoint to authelia, both by name and verified. The collector
# library dials no unix socket itself, so the relay is needed.
#
# The client secret is fetched on the host, under the responder's store
# identity, and bound in read-only: the container cannot reach the store.
forwarderClientId = cfg.otel.clientId;
gatewayPortSuffix = lib.optionalString (
gatewayCfg.httpsPort != 443
) ":${toString gatewayCfg.httpsPort}";
otelDomain = hyperhiveCfg.swarm.otel.domain;
autheliaDomain = hyperhiveCfg.swarm.authelia.domain;
# Its own directory, not `socketDir`: that one's mode admits nginx to the
# store's listener, and this one admits the container to nginx's.
egressDir = "/run/swarm-victoriametrics-egress";
egressSocket = "${egressDir}/egress.sock";
relayUnit = "swarm-victoriametrics-egress-relay";
# Loopback in the container's own netns.
relayPort = 4380;
relayUrl = "http://127.0.0.1:${toString relayPort}";
forwarderTelemetryPort = 8900;
# The forwarder's way out, see `egressSocket`. Every other path is refused,
# so the container reaches the two routes its forwarder uses and nothing
# else on the network.
egressServer = ''
server {
listen unix:${egressSocket};
location / {
return 403;
}
location ^~ /${cfg.machine}/ {
proxy_pass https://${otelDomain}${gatewayPortSuffix};
${gatewayCfg.lib.verifiedProxyTo otelDomain}
}
location = /api/oidc/token {
proxy_pass https://${autheliaDomain};
${gatewayCfg.lib.verifiedProxyTo autheliaDomain}
}
}
'';
forwarderFetchUnit = "swarm-victoriametrics-forwarder-oidc";
forwarderStoreSecretPath = "secret/swarm/services/${forwarderClientId}/oidc/client";
# On the host's /run (tmpfs), and seen read-only in the container at the
# same path. The file is not created ahead of the fetch: `LoadCredential`
# refuses a missing source, so the forwarder stays down until the store
# answers and then resumes from its saved read cursor.
forwarderSecretDir = "/run/${forwarderFetchUnit}";
forwarderSecret = "${forwarderSecretDir}/${forwarderClientId}.secret";
storeRetry = import ./lib/store-retry.nix { };
storeRefreshTimer = import ./lib/store-refresh-timer.nix { };
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
refreshConsumer = import ./lib/refresh-consumer.nix { };
in
{
# What the store IS from any hive's point of view, the name it answers on,
@ -228,6 +295,16 @@ in
both options.
'';
}
{
assertion =
(hyperhiveCfg.swarm.otel.serviceProducers.${cfg.machine}.clientId or null) == forwarderClientId;
message = ''
services.hyperhive.swarm.otel.serviceProducers has no entry
"${cfg.machine}" with clientId "${forwarderClientId}", so the swarm
collector has no receiver for the metrics store's journal forwarder.
Declare it in swarm-otel-service.nix.
'';
}
];
# The gateway name, inside `deployCfg.victoriametrics.enable` — that
@ -244,8 +321,9 @@ in
# Which check a read location's `auth_request` gets. `auth_request` takes a
# fixed URI, so it names `/__metrics_read_auth`, which rewrites to the
# target this map picks: a bearer goes to Grafana's check, anything else
# (an operator's session cookie) to authelia.
services.nginx.appendHttpConfig = ''
# (an operator's session cookie) to authelia. After the forwarder's egress
# listener, `egressServer`.
services.nginx.appendHttpConfig = egressServer + ''
map $http_authorization $swarm_metrics_read_auth_target {
"~^Bearer " /__metrics_grafana_auth;
default /__metrics_authelia;
@ -359,6 +437,12 @@ in
# Everything else is kept out.
systemd.tmpfiles.rules = [
"d ${socketDir} 0770 root ${toString nginxGid} - -"
# The relay in the container connects as root; nothing else on the host
# but nginx may.
"d ${egressDir} 0750 ${config.services.nginx.user} ${config.services.nginx.group} - -"
# The forwarder secret's bind source, so it exists when nspawn sets the
# mount up. Not the file: see `forwarderSecretDir`.
"d ${forwarderSecretDir} 0700 root root -"
];
# Grafana's check on the read API. On the host rather than in the store's
@ -422,12 +506,100 @@ in
};
};
# nginx runs as its own user under `ProtectSystem=strict`, so it can only
# create the socket where it may write. It removes the socket on a clean
# exit only; a stale one after a crash fails every later `bind`, and with
# it the whole gateway, so it goes before each start. `-`: the directory
# is tmpfiles', and absent until that has run.
systemd.services.nginx = {
serviceConfig.ReadWritePaths = [ "-${egressDir}" ];
preStart = lib.mkBefore "rm -f ${egressSocket}";
};
# Reads the forwarder's client secret with the responder's store identity,
# the shape ./swarm-bao.nix's `swarm-bao-forwarder-oidc` has. The store
# and authelia may be a host away and this boot waits on neither, so a run
# that fails retries (./lib/store-retry.nix), and the collector restarts
# until the file lands.
systemd.services.${forwarderFetchUnit} = {
description = "fetch the metrics store forwarder's OIDC client secret from the swarm secret store";
after = [
"swarm-bao-pki.service"
"container@${cfg.machine}.service"
];
wants = [ "container@${cfg.machine}.service" ];
wantedBy = [ "multi-user.target" ];
path = [
deployCfg.bao.package
pkgs.coreutils
pkgs.systemd
];
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
# No `RemainAfterExit`: the timer below starts this again, and
# starting an active unit is a no-op.
Type = "oneshot";
SyslogIdentifier = forwarderFetchUnit;
TimeoutStartSec = 30;
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = deployCfg.victoriametrics.authBaoClientCertFile;
BAO_CLIENT_KEY = deployCfg.victoriametrics.authBaoClientKeyFile;
}
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
BAO_CACERT = deployCfg.bao.serverCaFile;
};
script = ''
set -euo pipefail
${atomicWriteSecret}
${refreshConsumer}
err="$(mktemp)"
trap 'rm -f "$err"' EXIT
# `-token-only` answers on stdout and skips the token helper, which
# needs a `sh` this unit's `path` does not carry.
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to the swarm secret store with the metrics store responder's certificate." >&2
cat "$err" >&2
exit 1
fi
export BAO_TOKEN
if ! secret="$(bao kv get -field=value ${lib.escapeShellArg forwarderStoreSecretPath} 2>"$err")"; then
echo "the store did not return ${forwarderStoreSecretPath}: the forwarder has no client secret yet." >&2
cat "$err" >&2
exit 1
fi
if [ -z "$secret" ]; then
echo "the store returned an empty ${forwarderStoreSecretPath}." >&2
exit 1
fi
# `printf` inside `atomic_write_secret` is bash's own, so the value
# never becomes an argument in /proc.
install -d -m 0700 ${lib.escapeShellArg forwarderSecretDir}
if secret_differs ${lib.escapeShellArg forwarderSecret} "$secret"; then
atomic_write_secret 0400 root:root ${lib.escapeShellArg forwarderSecret} "$secret"
fi
# `LoadCredential` copies the file at start only: a rotated secret
# reaches the forwarder by restarting it.
refresh_consumer ${lib.escapeShellArg cfg.machine} opentelemetry-collector.service ${lib.escapeShellArg forwarderSecret}
'';
};
systemd.timers.${forwarderFetchUnit} =
storeRefreshTimer "re-read the metrics store forwarder's OIDC client secret from the swarm secret store";
containers.${cfg.machine} = {
autoStart = true;
ephemeral = false;
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [ "--link-journal=host" ];
# No `--link-journal=host`: the journal stays in the guest, where the
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
inherit privateNetwork;
# A bind mount rather than a path both sides happen to know: a
@ -436,11 +608,54 @@ in
hostPath = socketDir;
isReadOnly = false;
};
# Connecting to a socket is a write to it.
bindMounts.${egressDir} = {
hostPath = egressDir;
isReadOnly = false;
};
bindMounts.${forwarderSecretDir} = {
hostPath = forwarderSecretDir;
isReadOnly = true;
};
config =
{ config, pkgs, ... }:
{
imports = [ ../container-modules/swarm-container.nix ];
imports = [
../container-modules/swarm-container.nix
../container-modules/swarm-journal-forwarder.nix
];
# Authenticated as the forwarder's own OIDC client, its secret
# fetched on the host. Both URLs are the relay's.
services.hyperhive.swarmJournalForwarder = {
enable = true;
serviceName = "victoriametrics";
endpoint = "${relayUrl}/${cfg.machine}";
clientId = forwarderClientId;
tokenUrl = "${relayUrl}/api/oidc/token";
telemetryPort = forwarderTelemetryPort;
hostSecretFile = forwarderSecret;
};
# The relay out of this netns, started by its first connection.
# systemd-socket-proxyd as root: the egress directory admits root
# and nginx only.
systemd.sockets.${relayUnit} = {
description = "loopback listener for the journal forwarder's way out of this container";
wantedBy = [ "sockets.target" ];
listenStreams = [ "127.0.0.1:${toString relayPort}" ];
};
systemd.services.${relayUnit} = {
description = "relay the journal forwarder's connections to the host's egress socket";
requires = [ "${relayUnit}.socket" ];
after = [ "${relayUnit}.socket" ];
serviceConfig = {
ExecStart = "${pkgs.systemd}/lib/systemd/systemd-socket-proxyd ${egressSocket}";
PrivateTmp = true;
NoNewPrivileges = true;
};
};
# No resolver unit: with no interface but loopback there is no
# nameserver to reach, and the store resolves no names.

12
nix/module-eval/bao-grants.nix
View file

@ -781,9 +781,11 @@ let
lib.hasInfix "/logs-auth.pem ]" s && lib.hasInfix "swarm-logs-auth \"\" clientAuth" s;
}
{
# Read-only on the one leaf Grafana's host creates, and nothing else:
# this principal checks a presented bearer and does nothing with it.
name = "the metrics store responder's grant is Grafana's metrics bearer, read-only, and nothing else";
# Read-only on the one leaf Grafana's host creates and on the store
# container forwarder's client secret, and nothing else: this principal
# checks a presented bearer, and its host fetches one secret for the
# container.
name = "the metrics store responder's grant is Grafana's metrics bearer and its container forwarder's client secret, read-only, and nothing else";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-metrics-auth-policy.script;
@ -793,6 +795,10 @@ let
path = "secret/data/swarm/services/swarm-grafana/metrics/bearer";
caps = [ "read" ];
}
{
path = "secret/data/swarm/services/swarm-victoriametrics-collector/oidc/client";
caps = [ "read" ];
}
]
&& lib.hasInfix "auth/cert/certs/swarm-metrics-auth" s
&& lib.hasInfix "allowed_common_names=swarm-metrics-auth" s

70
nix/module-eval/nats-authelia.nix
View file

@ -205,6 +205,10 @@ let
machine = "hive-matrix";
option = "matrix";
}
{
machine = "swarm-victoriametrics";
option = "victoriametrics";
}
];
ownClientIdOf = h: svc: h.services.hyperhive.swarm.${svc.option}.otel.clientId;
@ -954,6 +958,72 @@ let
ok =
!shared.matrixNoBaoIdentity.containers.hive-matrix.config.services.opentelemetry-collector.enable;
}
{
# The metrics store's container has no network: the forwarder dials the
# relay on loopback and reads a secret the host fetched and bound in.
name = "the metrics store container forwards its journal through the relay as its own client";
ok =
let
h = servicesHost;
container = h.containers.swarm-victoriametrics;
c = container.config;
s = c.services.opentelemetry-collector.settings;
otel = h.services.hyperhive.swarm.otel;
clientId = h.services.hyperhive.swarm.victoriametrics.otel.clientId;
secret = "/run/swarm-victoriametrics-forwarder-oidc/${clientId}.secret";
fetch = h.systemd.services.swarm-victoriametrics-forwarder-oidc;
in
c.services.opentelemetry-collector.enable
&& s.exporters.otlphttp.endpoint == "http://127.0.0.1:4380/swarm-victoriametrics"
&& s.extensions."oauth2client/swarm".token_url == "http://127.0.0.1:4380/api/oidc/token"
&& s.extensions."oauth2client/swarm".client_id == clientId
&& s.extensions."oauth2client/swarm".endpoint_params.audience == clientId
&& otel.serviceProducers.swarm-victoriametrics.clientId == clientId
&&
(lib.findFirst (a: a.key == "service.name") null s.processors.resource.attributes).value
== "victoriametrics"
&& lib.elem "oidc-client-secret:${secret}" c.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
&& !(c.systemd.services ? swarm-journal-forwarder-oidc)
&& container.bindMounts."/run/swarm-victoriametrics-forwarder-oidc".isReadOnly
&& !(lib.elem "--link-journal=host" (container.extraFlags or [ ]))
&&
fetch.environment.BAO_CLIENT_CERT
== h.services.hyperhive.deploy.victoriametrics.authBaoClientCertFile
&&
fetch.environment.BAO_CLIENT_KEY == h.services.hyperhive.deploy.victoriametrics.authBaoClientKeyFile
&& lib.hasInfix "bao kv get -field=value secret/swarm/services/${clientId}/oidc/client" fetch.script
&& lib.hasInfix "refresh_consumer swarm-victoriametrics opentelemetry-collector.service" fetch.script;
}
{
# The relay: a loopback listener in the container, a proxy to the bound
# socket, and the host nginx server on that socket, which opens the
# forwarder's two routes and nothing else.
name = "the metrics store's egress relay leads loopback to the collector's route and authelia's token endpoint only";
ok =
let
h = servicesHost;
container = h.containers.swarm-victoriametrics;
c = container.config;
sock = "/run/swarm-victoriametrics-egress/egress.sock";
relay = "swarm-victoriametrics-egress-relay";
nginx = h.services.nginx.appendHttpConfig;
otelDomain = h.services.hyperhive.swarm.otel.domain;
autheliaDomain = h.services.hyperhive.swarm.authelia.domain;
in
c.systemd.sockets.${relay}.listenStreams == [ "127.0.0.1:4380" ]
&&
lib.hasSuffix "/lib/systemd/systemd-socket-proxyd ${sock}"
c.systemd.services.${relay}.serviceConfig.ExecStart
&& !container.bindMounts."/run/swarm-victoriametrics-egress".isReadOnly
&& lib.hasInfix "listen unix:${sock};" nginx
&& lib.hasInfix "location / {\n return 403;" nginx
&& lib.hasInfix "location ^~ /swarm-victoriametrics/ {\n proxy_pass https://${otelDomain}" nginx
&& lib.hasInfix "proxy_ssl_name ${otelDomain};" nginx
&& lib.hasInfix "location = /api/oidc/token {\n proxy_pass https://${autheliaDomain};" nginx
&& lib.hasInfix "proxy_ssl_name ${autheliaDomain};" nginx
&& lib.elem "-/run/swarm-victoriametrics-egress" h.systemd.services.nginx.serviceConfig.ReadWritePaths
&& lib.hasInfix "rm -f ${sock}" h.systemd.services.nginx.preStart;
}
]
++ forwarderCases
++ hostNetnsForwarderCases;

5
nix/module-eval/secret-publisher.nix
View file

@ -219,8 +219,8 @@ let
{
# The forwarders in the CI runner's, Grafana's, the log store's, the
# forge's and the homeserver's containers read their secrets from these
# paths.
name = "the publisher carries the CI runner, Grafana, log store, forge and homeserver forwarders' secrets";
# paths, and the metrics store's host reads its container's.
name = "the publisher carries the CI runner, Grafana, log store, forge, homeserver and metrics store forwarders' secrets";
ok =
lib.all
(
@ -235,6 +235,7 @@ let
"swarm-victorialogs-collector"
"swarm-forge-collector"
"swarm-matrix-collector"
"swarm-victoriametrics-collector"
];
}
{