vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-victoriametrics: forward the container's journal to the swarm collector

Browse source 
The metrics store's container now runs the journal forwarder the other
service containers run, pushing to a receiver of its own on the swarm
collector (serviceProducers swarm-victoriametrics) as a new machine OIDC
client (swarm.victoriametrics.otel.clientId), registered by a glue file
wherever authelia runs and published to the store by the secret publisher.

The container keeps its loopback-only netns. The collector cannot dial a
unix socket, so swarm-victoriametrics-egress-relay, a socket-activated
systemd-socket-proxyd inside the container, takes its connections on
127.0.0.1:4380 and passes them to /run/swarm-victoriametrics-egress/egress.sock.
The host's nginx listens there and sends /swarm-victoriametrics/ to the swarm
collector and /api/oidc/token to authelia, both by name and verified, and
refuses every other path. nginx gets ReadWritePaths on that directory and
removes a stale socket before it starts.

The container cannot reach the store, so the client secret is fetched on the
host by swarm-victoriametrics-forwarder-oidc under swarm-metrics-auth's leaf,
the shape swarm-bao-forwarder-oidc has, and bound into the container
read-only. The forwarder module gains hostSecretFile for that: set, it renders
no fetch unit. swarm-metrics-auth's policy gains one read stanza on the
forwarder's client path.

The container drops --link-journal=host, and the host collector stops reading
it.

Also lists the forge publisher's and matrix-ctl's leaves in the forwarder
secrets row of docs/swarm/secrets.md, which the hive-forge/hive-matrix change
missed.
This commit is contained in:
atlas 2026-10-16 12:05:10 +02:00 • committed by mara
commit 7294707d7d
15 changed files with 454 additions and 62 deletions
Download patch file Download diff file Expand all files Collapse all files

12
nix/module-eval/bao-grants.nix
View file

@ -781,9 +781,11 @@ let
lib.hasInfix "/logs-auth.pem ]" s && lib.hasInfix "swarm-logs-auth \"\" clientAuth" s;
}
{
# Read-only on the one leaf Grafana's host creates, and nothing else:
# this principal checks a presented bearer and does nothing with it.
name = "the metrics store responder's grant is Grafana's metrics bearer, read-only, and nothing else";
# Read-only on the one leaf Grafana's host creates and on the store
# container forwarder's client secret, and nothing else: this principal
# checks a presented bearer, and its host fetches one secret for the
# container.
name = "the metrics store responder's grant is Grafana's metrics bearer and its container forwarder's client secret, read-only, and nothing else";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-metrics-auth-policy.script;
@ -793,6 +795,10 @@ let
path = "secret/data/swarm/services/swarm-grafana/metrics/bearer";
caps = [ "read" ];
}
{
path = "secret/data/swarm/services/swarm-victoriametrics-collector/oidc/client";
caps = [ "read" ];
}
]
&& lib.hasInfix "auth/cert/certs/swarm-metrics-auth" s
&& lib.hasInfix "allowed_common_names=swarm-metrics-auth" s

70
nix/module-eval/nats-authelia.nix
View file

@ -205,6 +205,10 @@ let
machine = "hive-matrix";
option = "matrix";
}
{
machine = "swarm-victoriametrics";
option = "victoriametrics";
}
];
ownClientIdOf = h: svc: h.services.hyperhive.swarm.${svc.option}.otel.clientId;
@ -954,6 +958,72 @@ let
ok =
!shared.matrixNoBaoIdentity.containers.hive-matrix.config.services.opentelemetry-collector.enable;
}
{
# The metrics store's container has no network: the forwarder dials the
# relay on loopback and reads a secret the host fetched and bound in.
name = "the metrics store container forwards its journal through the relay as its own client";
ok =
let
h = servicesHost;
container = h.containers.swarm-victoriametrics;
c = container.config;
s = c.services.opentelemetry-collector.settings;
otel = h.services.hyperhive.swarm.otel;
clientId = h.services.hyperhive.swarm.victoriametrics.otel.clientId;
secret = "/run/swarm-victoriametrics-forwarder-oidc/${clientId}.secret";
fetch = h.systemd.services.swarm-victoriametrics-forwarder-oidc;
in
c.services.opentelemetry-collector.enable
&& s.exporters.otlphttp.endpoint == "http://127.0.0.1:4380/swarm-victoriametrics"
&& s.extensions."oauth2client/swarm".token_url == "http://127.0.0.1:4380/api/oidc/token"
&& s.extensions."oauth2client/swarm".client_id == clientId
&& s.extensions."oauth2client/swarm".endpoint_params.audience == clientId
&& otel.serviceProducers.swarm-victoriametrics.clientId == clientId
&&
(lib.findFirst (a: a.key == "service.name") null s.processors.resource.attributes).value
== "victoriametrics"
&& lib.elem "oidc-client-secret:${secret}" c.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential
&& !(c.systemd.services ? swarm-journal-forwarder-oidc)
&& container.bindMounts."/run/swarm-victoriametrics-forwarder-oidc".isReadOnly
&& !(lib.elem "--link-journal=host" (container.extraFlags or [ ]))
&&
fetch.environment.BAO_CLIENT_CERT
== h.services.hyperhive.deploy.victoriametrics.authBaoClientCertFile
&&
fetch.environment.BAO_CLIENT_KEY == h.services.hyperhive.deploy.victoriametrics.authBaoClientKeyFile
&& lib.hasInfix "bao kv get -field=value secret/swarm/services/${clientId}/oidc/client" fetch.script
&& lib.hasInfix "refresh_consumer swarm-victoriametrics opentelemetry-collector.service" fetch.script;
}
{
# The relay: a loopback listener in the container, a proxy to the bound
# socket, and the host nginx server on that socket, which opens the
# forwarder's two routes and nothing else.
name = "the metrics store's egress relay leads loopback to the collector's route and authelia's token endpoint only";
ok =
let
h = servicesHost;
container = h.containers.swarm-victoriametrics;
c = container.config;
sock = "/run/swarm-victoriametrics-egress/egress.sock";
relay = "swarm-victoriametrics-egress-relay";
nginx = h.services.nginx.appendHttpConfig;
otelDomain = h.services.hyperhive.swarm.otel.domain;
autheliaDomain = h.services.hyperhive.swarm.authelia.domain;
in
c.systemd.sockets.${relay}.listenStreams == [ "127.0.0.1:4380" ]
&&
lib.hasSuffix "/lib/systemd/systemd-socket-proxyd ${sock}"
c.systemd.services.${relay}.serviceConfig.ExecStart
&& !container.bindMounts."/run/swarm-victoriametrics-egress".isReadOnly
&& lib.hasInfix "listen unix:${sock};" nginx
&& lib.hasInfix "location / {\n return 403;" nginx
&& lib.hasInfix "location ^~ /swarm-victoriametrics/ {\n proxy_pass https://${otelDomain}" nginx
&& lib.hasInfix "proxy_ssl_name ${otelDomain};" nginx
&& lib.hasInfix "location = /api/oidc/token {\n proxy_pass https://${autheliaDomain};" nginx
&& lib.hasInfix "proxy_ssl_name ${autheliaDomain};" nginx
&& lib.elem "-/run/swarm-victoriametrics-egress" h.systemd.services.nginx.serviceConfig.ReadWritePaths
&& lib.hasInfix "rm -f ${sock}" h.systemd.services.nginx.preStart;
}
]
++ forwarderCases
++ hostNetnsForwarderCases;

5
nix/module-eval/secret-publisher.nix
View file

@ -219,8 +219,8 @@ let
{
# The forwarders in the CI runner's, Grafana's, the log store's, the
# forge's and the homeserver's containers read their secrets from these
# paths.
name = "the publisher carries the CI runner, Grafana, log store, forge and homeserver forwarders' secrets";
# paths, and the metrics store's host reads its container's.
name = "the publisher carries the CI runner, Grafana, log store, forge, homeserver and metrics store forwarders' secrets";
ok =
lib.all
(
@ -235,6 +235,7 @@ let
"swarm-victorialogs-collector"
"swarm-forge-collector"
"swarm-matrix-collector"
"swarm-victoriametrics-collector"
];
}
{