vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

hive-forge, hive-matrix: forward each container's journal to the swarm collector

Browse source 
Each container now runs the journal forwarder the CI runner's, Grafana's and
the log store's containers run, pushing to a receiver of its own on the swarm
collector (serviceProducers hive-forge and hive-matrix). Each authenticates as
a new machine OIDC client of its own (swarm.forge.otel.clientId,
swarm.matrix.otel.clientId), registered by a glue file wherever authelia runs,
minted by authelia and published to the store by the secret publisher. The
forge's and the homeserver's browser login clients are unchanged. The client
ids are swarm-forge-collector and swarm-matrix-collector, not
hive-<x>-collector, which is the client id a hive named <x>-collector gets.

The fetch unit inside each container reads the client secret with the store
identity the host already holds for that service. The forge publisher's leaf
is handed into the forge container as an nspawn system credential, as the
grafana and log store containers do. matrix-ctl's leaf is already bound into
the homeserver's container, so its forwarder reads it at those paths, as
swarm-matrix-appservice-publish does; with no matrix-ctl identity the
homeserver's forwarder is off. Each of the two policies gains one read stanza
on its forwarder's client path. The host units that use those identities are
unchanged.

The two containers drop --link-journal=host, and the host collector stops
reading them.
This commit is contained in:
atlas 2026-10-16 11:48:22 +02:00 • committed by mara
commit ce44b6e77e
18 changed files with 511 additions and 134 deletions
Download patch file Download diff file Expand all files Collapse all files

58
nix/module-eval/bao-grants.nix
View file

@ -517,17 +517,33 @@ let
# read as tidier.
#
# The first leaf is the swarm appservice token, which matrix-ctl mints
# and publishes for the controller; the second is the homeserver's own
# OIDC client secret, read-only. Counted, so a third stanza fails rather
# than riding along beside correct ones.
name = "matrix-ctl's grant is the swarm appservice token and the homeserver's OIDC secret, nothing else";
# and publishes for the controller; the second and third are the
# homeserver's own OIDC client secret and its container forwarder's,
# read-only. The whole list, so a fourth stanza fails rather than riding
# along beside correct ones.
name = "matrix-ctl's grant is the swarm appservice token and the homeserver's and its forwarder's OIDC secrets, nothing else";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-matrix-ctl-policy.script;
in
lib.hasInfix "path \"secret/data/swarm/controller/swarm-controller/matrix/appservice-token\" {\n capabilities = [\"create\", \"update\", \"read\"]\n}" s
&& lib.hasInfix "path \"secret/data/swarm/services/tuwunel/oidc/client\" {\n capabilities = [\"read\"]\n}" s
&& lib.length (lib.splitString "path \"" s) == 3
grantsIn s == [
{
path = "secret/data/swarm/controller/swarm-controller/matrix/appservice-token";
caps = [
"create"
"update"
"read"
];
}
{
path = "secret/data/swarm/services/tuwunel/oidc/client";
caps = [ "read" ];
}
{
path = "secret/data/swarm/services/swarm-matrix-collector/oidc/client";
caps = [ "read" ];
}
]
&& !(lib.hasInfix "secret/data/swarm/services/*" s)
&& !(lib.hasInfix "swarm-grafana" s)
&& !(lib.hasInfix "secret/data/swarm/agents" s)
@ -1715,17 +1731,31 @@ let
}
{
# The forge's host is the one writer of the controller's forge token,
# and reads the forge's own OIDC client secret under the same leaf.
# Pinned as two whole stanzas, so an added `read` on the token or a
# write or wildcard on the secret fails.
name = "the forge publisher's grant is writing the controller's forge token and reading the forge's OIDC secret, and nothing else";
# and reads the forge's own OIDC client secret and its container
# forwarder's under the same leaf. Pinned as the whole list, so an added
# `read` on the token or a write or wildcard on a secret fails.
name = "the forge publisher's grant is writing the controller's forge token and reading the forge's and its forwarder's OIDC secrets, and nothing else";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-forge-publisher-policy.script;
in
lib.hasInfix "path \"secret/data/swarm/controller/swarm-controller/forge/token\" {\n capabilities = [\"create\", \"update\"]\n}" s
&& lib.hasInfix "path \"secret/data/swarm/services/forgejo/oidc/client\" {\n capabilities = [\"read\"]\n}" s
&& lib.length (lib.filter lib.isList (builtins.split "path \"" s)) == 2
grantsIn s == [
{
path = "secret/data/swarm/controller/swarm-controller/forge/token";
caps = [
"create"
"update"
];
}
{
path = "secret/data/swarm/services/forgejo/oidc/client";
caps = [ "read" ];
}
{
path = "secret/data/swarm/services/swarm-forge-collector/oidc/client";
caps = [ "read" ];
}
]
&& !(lib.hasInfix "secret/data/swarm/services/*" s)
&& lib.hasInfix "auth/cert/certs/swarm-forge-publisher" s
&& lib.hasInfix "allowed_common_names=swarm-forge-publisher" s

2
nix/module-eval/name-guards.nix
View file

@ -53,7 +53,7 @@ let
};
# The THIRD element of the same list, colliding on its own so neither of the
# two above can carry it. matrix-ctl's grant is one path rather than a whole
# two above can carry it. matrix-ctl's grant names its leaves, not a whole
# prefix, which is exactly why a dead entry here would be easy to miss: a
# hive that inherited it would not obviously break anything, it would
# silently gain the ability to overwrite the swarm's matrix credential.

61
nix/module-eval/nats-authelia.nix
View file

@ -182,7 +182,8 @@ let
# The forwarders that authenticate as a client of their own rather than the
# service's, each with the store identity the host already holds. The CI
# runner's container cases are in ./forge-placement.nix, beside its fixture.
# runner's container cases are in ./forge-placement.nix, beside its fixture;
# the homeserver's, which reads a bound-in identity, are below.
ownClientForwarders = [
{
machine = "swarm-ci-runner";
@ -196,10 +197,18 @@ let
machine = "swarm-victorialogs";
option = "victorialogs";
}
{
machine = "hive-forge";
option = "forge";
}
{
machine = "hive-matrix";
option = "matrix";
}
];
ownClientIdOf = h: svc: h.services.hyperhive.swarm.${svc.option}.otel.clientId;
# The two of them that share the host netns, on the shared services host.
# Those that share the host netns, on the shared services host.
hostNetnsForwarders = [
{
machine = "swarm-grafana";
@ -217,6 +226,14 @@ let
key = "bao-client-key.pem";
port = 8897;
}
{
machine = "hive-forge";
option = "forge";
name = "forgejo";
cert = "forge-bao-cert";
key = "forge-bao-key";
port = 8898;
}
];
hostNetnsForwarderCases = lib.concatMap (
@ -856,7 +873,7 @@ let
lib.all ok1 (map (svc: svc.machine) forwarderServices);
}
{
name = "the swarm collector admits the CI runner, Grafana and log store forwarders on receivers of their own";
name = "the swarm collector admits the CI runner, Grafana, log store, forge and homeserver forwarders on receivers of their own";
ok =
let
h = shared.baoWithCollector;
@ -877,7 +894,7 @@ let
{
# Grafana's forwarder is a second client beside its browser login, which
# stays an interactive client.
name = "the CI runner, Grafana and log store forwarder clients are registered as machine clients with their own audience and a signed access token";
name = "the CI runner, Grafana, log store, forge and homeserver forwarder clients are registered as machine clients with their own audience and a signed access token";
ok =
let
h = shared.autheliaOnly;
@ -901,6 +918,42 @@ let
&& browser != null
&& browser.kind != "machine";
}
{
# The homeserver's container already holds matrix-ctl's identity, bound
# in read-only, and its forwarder fetches with those paths rather than a
# system credential.
name = "the homeserver container forwards its journal to its own route as its own client";
ok =
let
h = servicesHost;
container = h.containers.hive-matrix;
c = container.config;
s = c.services.opentelemetry-collector.settings;
otel = h.services.hyperhive.swarm.otel;
fetch = c.systemd.services.swarm-journal-forwarder-oidc;
clientId = h.services.hyperhive.swarm.matrix.otel.clientId;
ctlCert = h.services.hyperhive.deploy.matrix.ctlBaoClientCertFile;
in
c.services.opentelemetry-collector.enable
&& s.exporters.otlphttp.endpoint == "https://${otel.domain}/hive-matrix"
&& s.extensions."oauth2client/swarm".client_id == clientId
&& s.extensions."oauth2client/swarm".endpoint_params.audience == clientId
&& otel.serviceProducers.hive-matrix.clientId == clientId
&&
(lib.findFirst (a: a.key == "service.name") null s.processors.resource.attributes).value
== "tuwunel"
&& fetch.environment.BAO_CLIENT_CERT == ctlCert
&& fetch.environment.BAO_CLIENT_KEY == h.services.hyperhive.deploy.matrix.ctlBaoClientKeyFile
&& container.bindMounts ? ${builtins.dirOf ctlCert}
&& lib.hasInfix "bao kv get -field=value secret/swarm/services/${clientId}/oidc/client" fetch.script
&& !(lib.elem "--link-journal=host" container.extraFlags);
}
{
# With no matrix-ctl identity there is nothing to fetch the secret with.
name = "a homeserver container with no store identity runs no forwarder";
ok =
!shared.matrixNoBaoIdentity.containers.hive-matrix.config.services.opentelemetry-collector.enable;
}
]
++ forwarderCases
++ hostNetnsForwarderCases;

9
nix/module-eval/secret-publisher.nix
View file

@ -217,9 +217,10 @@ let
);
}
{
# The forwarders in the CI runner's, Grafana's and the log store's
# containers read their secrets from these paths.
name = "the publisher carries the CI runner, Grafana and log store forwarders' secrets";
# The forwarders in the CI runner's, Grafana's, the log store's, the
# forge's and the homeserver's containers read their secrets from these
# paths.
name = "the publisher carries the CI runner, Grafana, log store, forge and homeserver forwarders' secrets";
ok =
lib.all
(
@ -232,6 +233,8 @@ let
"swarm-ci-runner-collector"
"swarm-grafana-collector"
"swarm-victorialogs-collector"
"swarm-forge-collector"
"swarm-matrix-collector"
];
}
{