swarm-ci-runner, swarm-grafana, swarm-victorialogs: forward each container's journal to the swarm collector
Each container now runs the journal forwarder the queue's and the builder's containers run, pushing to a receiver of its own on the swarm collector. Each authenticates as a new machine OIDC client of its own (swarm.<service>.otel.clientId), registered by a glue file wherever authelia runs, minted by authelia and published to the store by the secret publisher. Grafana's browser login client is unchanged. The fetch unit inside each container reads the client secret with the store identity the host already holds for that service (the runner's, Grafana's, and the log store responder's), now also handed into the container as an nspawn system credential. Each of those three policies gains one read stanza on its forwarder's client path. The host units that use those identities are unchanged. The three containers drop --link-journal=host, and the host collector stops reading them.
This commit is contained in:
atlas
mara
parent
c5dbab3277
commit
c6ca9ab845
20 changed files with 810 additions and 107 deletions
|
|
@ -180,6 +180,98 @@ let
|
|||
]
|
||||
) forwarderServices;
|
||||
|
||||
# The forwarders that authenticate as a client of their own rather than the
|
||||
# service's, each with the store identity the host already holds. The CI
|
||||
# runner's container cases are in ./forge-placement.nix, beside its fixture.
|
||||
ownClientForwarders = [
|
||||
{
|
||||
machine = "swarm-ci-runner";
|
||||
option = "ci-runner";
|
||||
}
|
||||
{
|
||||
machine = "swarm-grafana";
|
||||
option = "grafana";
|
||||
}
|
||||
{
|
||||
machine = "swarm-victorialogs";
|
||||
option = "victorialogs";
|
||||
}
|
||||
];
|
||||
ownClientIdOf = h: svc: h.services.hyperhive.swarm.${svc.option}.otel.clientId;
|
||||
|
||||
# The two of them that share the host netns, on the shared services host.
|
||||
hostNetnsForwarders = [
|
||||
{
|
||||
machine = "swarm-grafana";
|
||||
option = "grafana";
|
||||
name = "grafana";
|
||||
cert = "grafana-bao-cert";
|
||||
key = "grafana-bao-key";
|
||||
port = 8896;
|
||||
}
|
||||
{
|
||||
machine = "swarm-victorialogs";
|
||||
option = "victorialogs";
|
||||
name = "victorialogs";
|
||||
cert = "bao-client.pem";
|
||||
key = "bao-client-key.pem";
|
||||
port = 8897;
|
||||
}
|
||||
];
|
||||
|
||||
hostNetnsForwarderCases = lib.concatMap (
|
||||
svc:
|
||||
let
|
||||
h = servicesHost;
|
||||
container = h.containers.${svc.machine};
|
||||
c = container.config;
|
||||
s = c.services.opentelemetry-collector.settings;
|
||||
otel = h.services.hyperhive.swarm.otel;
|
||||
fetch = c.systemd.services.swarm-journal-forwarder-oidc;
|
||||
clientId = ownClientIdOf h svc;
|
||||
in
|
||||
[
|
||||
{
|
||||
name = "the ${svc.name} container forwards its journal to its own route as its own client";
|
||||
ok =
|
||||
c.services.opentelemetry-collector.enable
|
||||
&& s.exporters.otlphttp.endpoint == "https://${otel.domain}/${svc.machine}"
|
||||
&& s.extensions."oauth2client/swarm".client_id == clientId
|
||||
&& s.extensions."oauth2client/swarm".endpoint_params.audience == clientId
|
||||
&& otel.serviceProducers.${svc.machine}.clientId == clientId
|
||||
&&
|
||||
(lib.findFirst (a: a.key == "service.name") null s.processors.resource.attributes).value == svc.name
|
||||
&& fetch.environment.BAO_CLIENT_CERT == "${forwarderFetchDir}/${svc.cert}"
|
||||
&& fetch.environment.BAO_CLIENT_KEY == "${forwarderFetchDir}/${svc.key}"
|
||||
&& lib.elem svc.cert fetch.serviceConfig.LoadCredential
|
||||
&& lib.elem svc.key fetch.serviceConfig.LoadCredential
|
||||
&& lib.hasInfix "bao kv get -field=value secret/swarm/services/${clientId}/oidc/client" fetch.script
|
||||
&& !(lib.elem "--link-journal=host" container.extraFlags)
|
||||
&& lib.any (lib.hasPrefix "--load-credential=${svc.cert}:") container.extraFlags
|
||||
&& lib.any (lib.hasPrefix "--load-credential=${svc.key}:") container.extraFlags;
|
||||
}
|
||||
{
|
||||
# Shared host netns: one port space with every other collector there.
|
||||
name = "the ${svc.name} forwarder's metrics port is free in the host netns and loopback";
|
||||
ok =
|
||||
let
|
||||
taken = [
|
||||
8888
|
||||
8889
|
||||
8890
|
||||
h.services.hyperhive.swarm.nats.otel.telemetryPort
|
||||
]
|
||||
++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) hostNetnsForwarders);
|
||||
reader = lib.head s.service.telemetry.metrics.readers;
|
||||
in
|
||||
c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port
|
||||
&& !(lib.elem svc.port taken)
|
||||
&& reader.pull.exporter.prometheus.host == "127.0.0.1"
|
||||
&& reader.pull.exporter.prometheus.port == svc.port;
|
||||
}
|
||||
]
|
||||
) hostNetnsForwarders;
|
||||
|
||||
cases = [
|
||||
{
|
||||
name = "authelia and nats declare no scrape target on a host with no collector";
|
||||
|
|
@ -763,7 +855,54 @@ let
|
|||
in
|
||||
lib.all ok1 (map (svc: svc.machine) forwarderServices);
|
||||
}
|
||||
{
|
||||
name = "the swarm collector admits the CI runner, Grafana and log store forwarders on receivers of their own";
|
||||
ok =
|
||||
let
|
||||
h = shared.baoWithCollector;
|
||||
otel = h.services.hyperhive.swarm.otel;
|
||||
s = h.containers.swarm-otel.config.services.opentelemetry-collector.settings;
|
||||
ports = map (p: p.port) (lib.attrValues otel.serviceProducers);
|
||||
in
|
||||
lib.all (
|
||||
svc:
|
||||
s.receivers."otlp/${svc.machine}".protocols.http.auth.authenticator == "oidc/${svc.machine}"
|
||||
&& s.extensions."oidc/${svc.machine}".audience == ownClientIdOf h svc
|
||||
&& lib.elem "oidc/${svc.machine}" s.service.extensions
|
||||
&& lib.elem "otlp/${svc.machine}" s.service.pipelines."logs/${otel.producerName}".receivers
|
||||
&& h.services.nginx.virtualHosts.${otel.domain}.locations ? "/${svc.machine}/"
|
||||
) ownClientForwarders
|
||||
&& lib.length (lib.unique ports) == lib.length ports;
|
||||
}
|
||||
{
|
||||
# Grafana's forwarder is a second client beside its browser login, which
|
||||
# stays an interactive client.
|
||||
name = "the CI runner, Grafana and log store forwarder clients are registered as machine clients with their own audience and a signed access token";
|
||||
ok =
|
||||
let
|
||||
h = shared.autheliaOnly;
|
||||
clients = h.services.hyperhive.swarm.authelia.oidc.clients;
|
||||
find = id: lib.findFirst (c: c.id == id) null clients;
|
||||
ok1 =
|
||||
svc:
|
||||
let
|
||||
id = ownClientIdOf h svc;
|
||||
c = find id;
|
||||
in
|
||||
c != null
|
||||
&& c.audience == [ id ]
|
||||
&& c.accessTokenSignedResponseAlg == "RS256"
|
||||
&& c.kind == "machine";
|
||||
browser = find h.services.hyperhive.swarm.grafana.oidc.clientId;
|
||||
in
|
||||
lib.all ok1 ownClientForwarders
|
||||
&&
|
||||
h.services.hyperhive.swarm.grafana.otel.clientId != h.services.hyperhive.swarm.grafana.oidc.clientId
|
||||
&& browser != null
|
||||
&& browser.kind != "machine";
|
||||
}
|
||||
]
|
||||
++ forwarderCases;
|
||||
++ forwarderCases
|
||||
++ hostNetnsForwarderCases;
|
||||
in
|
||||
runGroup "nats-authelia" cases
|
||||
|
|
|
|||
Loading…
Reference in a new issue