swarm-ci-runner, swarm-grafana, swarm-victorialogs: forward each container's journal to the swarm collector
Each container now runs the journal forwarder the queue's and the builder's containers run, pushing to a receiver of its own on the swarm collector. Each authenticates as a new machine OIDC client of its own (swarm.<service>.otel.clientId), registered by a glue file wherever authelia runs, minted by authelia and published to the store by the secret publisher. Grafana's browser login client is unchanged. The fetch unit inside each container reads the client secret with the store identity the host already holds for that service (the runner's, Grafana's, and the log store responder's), now also handed into the container as an nspawn system credential. Each of those three policies gains one read stanza on its forwarder's client path. The host units that use those identities are unchanged. The three containers drop --link-journal=host, and the host collector stops reading them.
This commit is contained in:
atlas
mara
parent
c5dbab3277
commit
c6ca9ab845
20 changed files with 810 additions and 107 deletions
|
|
@ -600,7 +600,7 @@ let
|
|||
# run rotate a value Grafana or the metrics store already holds. The
|
||||
# controller's Grafana token is the one leaf it may replace, which it
|
||||
# does when Grafana refuses the stored token.
|
||||
name = "the Grafana reader reads its own client secret, creates its admin password and metrics bearer, and writes the controller's Grafana token, and nothing else";
|
||||
name = "the Grafana reader reads its own and its forwarder's client secrets, creates its admin password and metrics bearer, and writes the controller's Grafana token, and nothing else";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-grafana-oidc-policy.script;
|
||||
|
|
@ -610,6 +610,10 @@ let
|
|||
path = "secret/data/swarm/services/swarm-grafana/oidc/client";
|
||||
caps = [ "read" ];
|
||||
}
|
||||
{
|
||||
path = "secret/data/swarm/services/swarm-grafana-collector/oidc/client";
|
||||
caps = [ "read" ];
|
||||
}
|
||||
{
|
||||
path = "secret/data/swarm/services/swarm-grafana/admin/password";
|
||||
caps = [
|
||||
|
|
@ -726,10 +730,10 @@ let
|
|||
lib.hasInfix "/nats-auth.pem ]" s && lib.hasInfix "swarm-nats-auth \"\" clientAuth" s;
|
||||
}
|
||||
{
|
||||
# Every agent's queue secret and Grafana's metrics bearer, both
|
||||
# read-only, and nothing else: this principal checks a presented secret
|
||||
# and does nothing with it.
|
||||
name = "the log store responder's grant is every agent's queue secret and Grafana's bearer, read-only, and nothing else";
|
||||
# Every agent's queue secret, Grafana's metrics bearer and the store
|
||||
# container forwarder's client secret, all read-only, and nothing else:
|
||||
# this principal checks a presented secret and does nothing with it.
|
||||
name = "the log store responder's grant is every agent's queue secret, Grafana's bearer and its container forwarder's client secret, read-only, and nothing else";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-logs-auth-policy.script;
|
||||
|
|
@ -743,6 +747,10 @@ let
|
|||
path = "secret/data/swarm/services/swarm-grafana/metrics/bearer";
|
||||
caps = [ "read" ];
|
||||
}
|
||||
{
|
||||
path = "secret/data/swarm/services/swarm-victorialogs-collector/oidc/client";
|
||||
caps = [ "read" ];
|
||||
}
|
||||
]
|
||||
&& lib.hasInfix "auth/cert/certs/swarm-logs-auth" s
|
||||
&& lib.hasInfix "allowed_common_names=swarm-logs-auth" s
|
||||
|
|
@ -1536,16 +1544,24 @@ let
|
|||
&& !(lib.hasInfix "secret/data/swarm/services/review-classifier/acp-provider" s);
|
||||
}
|
||||
{
|
||||
# The runner's role is the token's only reader. Its policy is the one
|
||||
# leaf, read-only, under the runner's own subject.
|
||||
name = "the CI runner's grant is its registration token and nothing else";
|
||||
# The runner's role is the token's only reader. Its policy is that leaf
|
||||
# and its container forwarder's client secret, each read-only, under the
|
||||
# runner's own subject.
|
||||
name = "the CI runner's grant is its registration token and its forwarder's client secret, and nothing else";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-ci-runner-policy.script;
|
||||
in
|
||||
lib.hasInfix "path \"secret/data/swarm/services/ci-runner/registration-token\" {" s
|
||||
&& lib.hasInfix "capabilities = [\"read\"]" s
|
||||
&& lib.length (lib.filter lib.isList (builtins.split "path \"" s)) == 1
|
||||
grantsIn s == [
|
||||
{
|
||||
path = "secret/data/swarm/services/ci-runner/registration-token";
|
||||
caps = [ "read" ];
|
||||
}
|
||||
{
|
||||
path = "secret/data/swarm/services/swarm-ci-runner-collector/oidc/client";
|
||||
caps = [ "read" ];
|
||||
}
|
||||
]
|
||||
&& !(lib.hasInfix "secret/data/swarm/services/*" s)
|
||||
&& !(lib.hasInfix "secret/data/swarm/agents" s)
|
||||
&& !(lib.hasInfix "secret/data/swarm/hives" s)
|
||||
|
|
|
|||
|
|
@ -492,6 +492,51 @@ let
|
|||
forgeSettings.service.DISABLE_REGISTRATION == true
|
||||
&& !(forgeSettings.service.ALLOW_ONLY_INTERNAL_REGISTRATION or false);
|
||||
}
|
||||
{
|
||||
# The runner's own store leaf, handed in as system credentials, reads the
|
||||
# forwarder's client secret; the host's token fetch keeps the same files.
|
||||
name = "the CI runner container forwards its journal to its own route as its own client";
|
||||
ok =
|
||||
let
|
||||
h = ciAlone;
|
||||
container = h.containers.swarm-ci-runner;
|
||||
c = container.config;
|
||||
s = c.services.opentelemetry-collector.settings;
|
||||
fetch = c.systemd.services.swarm-journal-forwarder-oidc;
|
||||
clientId = h.services.hyperhive.swarm.ci-runner.otel.clientId;
|
||||
dir = "/run/credentials/swarm-journal-forwarder-oidc.service";
|
||||
in
|
||||
c.services.opentelemetry-collector.enable
|
||||
&&
|
||||
s.exporters.otlphttp.endpoint == "https://${h.services.hyperhive.swarm.otel.domain}/swarm-ci-runner"
|
||||
&& s.extensions."oauth2client/swarm".client_id == clientId
|
||||
&& s.extensions."oauth2client/swarm".endpoint_params.audience == clientId
|
||||
&& h.services.hyperhive.swarm.otel.serviceProducers.swarm-ci-runner.clientId == clientId
|
||||
&&
|
||||
(lib.findFirst (a: a.key == "service.name") null s.processors.resource.attributes).value
|
||||
== "ci-runner"
|
||||
&& fetch.environment.BAO_CLIENT_CERT == "${dir}/ci-runner-bao-cert"
|
||||
&& fetch.environment.BAO_CLIENT_KEY == "${dir}/ci-runner-bao-key"
|
||||
&& lib.elem "ci-runner-bao-cert" fetch.serviceConfig.LoadCredential
|
||||
&& lib.elem "ci-runner-bao-key" fetch.serviceConfig.LoadCredential
|
||||
&& lib.hasInfix "bao kv get -field=value secret/swarm/services/${clientId}/oidc/client" fetch.script
|
||||
&& !(lib.elem "--link-journal=host" container.extraFlags)
|
||||
&& lib.elem "--load-credential=ci-runner-bao-cert:/etc/pki/bao-ci-runner.pem" container.extraFlags
|
||||
&& lib.elem "--load-credential=ci-runner-bao-key:/etc/pki/bao-ci-runner-key.pem" container.extraFlags
|
||||
&&
|
||||
h.systemd.services.swarm-bao-ci-runner.environment.BAO_CLIENT_CERT == "/etc/pki/bao-ci-runner.pem";
|
||||
}
|
||||
{
|
||||
name = "the CI runner forwarder's metrics port is loopback";
|
||||
ok =
|
||||
let
|
||||
c = ciAlone.containers.swarm-ci-runner.config;
|
||||
reader = lib.head c.services.opentelemetry-collector.settings.service.telemetry.metrics.readers;
|
||||
in
|
||||
c.services.hyperhive.swarmJournalForwarder.telemetryPort == 8895
|
||||
&& reader.pull.exporter.prometheus.host == "127.0.0.1"
|
||||
&& reader.pull.exporter.prometheus.port == 8895;
|
||||
}
|
||||
];
|
||||
in
|
||||
runGroup "forge-placement" cases
|
||||
|
|
|
|||
|
|
@ -180,6 +180,98 @@ let
|
|||
]
|
||||
) forwarderServices;
|
||||
|
||||
# The forwarders that authenticate as a client of their own rather than the
|
||||
# service's, each with the store identity the host already holds. The CI
|
||||
# runner's container cases are in ./forge-placement.nix, beside its fixture.
|
||||
ownClientForwarders = [
|
||||
{
|
||||
machine = "swarm-ci-runner";
|
||||
option = "ci-runner";
|
||||
}
|
||||
{
|
||||
machine = "swarm-grafana";
|
||||
option = "grafana";
|
||||
}
|
||||
{
|
||||
machine = "swarm-victorialogs";
|
||||
option = "victorialogs";
|
||||
}
|
||||
];
|
||||
ownClientIdOf = h: svc: h.services.hyperhive.swarm.${svc.option}.otel.clientId;
|
||||
|
||||
# The two of them that share the host netns, on the shared services host.
|
||||
hostNetnsForwarders = [
|
||||
{
|
||||
machine = "swarm-grafana";
|
||||
option = "grafana";
|
||||
name = "grafana";
|
||||
cert = "grafana-bao-cert";
|
||||
key = "grafana-bao-key";
|
||||
port = 8896;
|
||||
}
|
||||
{
|
||||
machine = "swarm-victorialogs";
|
||||
option = "victorialogs";
|
||||
name = "victorialogs";
|
||||
cert = "bao-client.pem";
|
||||
key = "bao-client-key.pem";
|
||||
port = 8897;
|
||||
}
|
||||
];
|
||||
|
||||
hostNetnsForwarderCases = lib.concatMap (
|
||||
svc:
|
||||
let
|
||||
h = servicesHost;
|
||||
container = h.containers.${svc.machine};
|
||||
c = container.config;
|
||||
s = c.services.opentelemetry-collector.settings;
|
||||
otel = h.services.hyperhive.swarm.otel;
|
||||
fetch = c.systemd.services.swarm-journal-forwarder-oidc;
|
||||
clientId = ownClientIdOf h svc;
|
||||
in
|
||||
[
|
||||
{
|
||||
name = "the ${svc.name} container forwards its journal to its own route as its own client";
|
||||
ok =
|
||||
c.services.opentelemetry-collector.enable
|
||||
&& s.exporters.otlphttp.endpoint == "https://${otel.domain}/${svc.machine}"
|
||||
&& s.extensions."oauth2client/swarm".client_id == clientId
|
||||
&& s.extensions."oauth2client/swarm".endpoint_params.audience == clientId
|
||||
&& otel.serviceProducers.${svc.machine}.clientId == clientId
|
||||
&&
|
||||
(lib.findFirst (a: a.key == "service.name") null s.processors.resource.attributes).value == svc.name
|
||||
&& fetch.environment.BAO_CLIENT_CERT == "${forwarderFetchDir}/${svc.cert}"
|
||||
&& fetch.environment.BAO_CLIENT_KEY == "${forwarderFetchDir}/${svc.key}"
|
||||
&& lib.elem svc.cert fetch.serviceConfig.LoadCredential
|
||||
&& lib.elem svc.key fetch.serviceConfig.LoadCredential
|
||||
&& lib.hasInfix "bao kv get -field=value secret/swarm/services/${clientId}/oidc/client" fetch.script
|
||||
&& !(lib.elem "--link-journal=host" container.extraFlags)
|
||||
&& lib.any (lib.hasPrefix "--load-credential=${svc.cert}:") container.extraFlags
|
||||
&& lib.any (lib.hasPrefix "--load-credential=${svc.key}:") container.extraFlags;
|
||||
}
|
||||
{
|
||||
# Shared host netns: one port space with every other collector there.
|
||||
name = "the ${svc.name} forwarder's metrics port is free in the host netns and loopback";
|
||||
ok =
|
||||
let
|
||||
taken = [
|
||||
8888
|
||||
8889
|
||||
8890
|
||||
h.services.hyperhive.swarm.nats.otel.telemetryPort
|
||||
]
|
||||
++ map (o: o.port) (lib.filter (o: o.machine != svc.machine) hostNetnsForwarders);
|
||||
reader = lib.head s.service.telemetry.metrics.readers;
|
||||
in
|
||||
c.services.hyperhive.swarmJournalForwarder.telemetryPort == svc.port
|
||||
&& !(lib.elem svc.port taken)
|
||||
&& reader.pull.exporter.prometheus.host == "127.0.0.1"
|
||||
&& reader.pull.exporter.prometheus.port == svc.port;
|
||||
}
|
||||
]
|
||||
) hostNetnsForwarders;
|
||||
|
||||
cases = [
|
||||
{
|
||||
name = "authelia and nats declare no scrape target on a host with no collector";
|
||||
|
|
@ -763,7 +855,54 @@ let
|
|||
in
|
||||
lib.all ok1 (map (svc: svc.machine) forwarderServices);
|
||||
}
|
||||
{
|
||||
name = "the swarm collector admits the CI runner, Grafana and log store forwarders on receivers of their own";
|
||||
ok =
|
||||
let
|
||||
h = shared.baoWithCollector;
|
||||
otel = h.services.hyperhive.swarm.otel;
|
||||
s = h.containers.swarm-otel.config.services.opentelemetry-collector.settings;
|
||||
ports = map (p: p.port) (lib.attrValues otel.serviceProducers);
|
||||
in
|
||||
lib.all (
|
||||
svc:
|
||||
s.receivers."otlp/${svc.machine}".protocols.http.auth.authenticator == "oidc/${svc.machine}"
|
||||
&& s.extensions."oidc/${svc.machine}".audience == ownClientIdOf h svc
|
||||
&& lib.elem "oidc/${svc.machine}" s.service.extensions
|
||||
&& lib.elem "otlp/${svc.machine}" s.service.pipelines."logs/${otel.producerName}".receivers
|
||||
&& h.services.nginx.virtualHosts.${otel.domain}.locations ? "/${svc.machine}/"
|
||||
) ownClientForwarders
|
||||
&& lib.length (lib.unique ports) == lib.length ports;
|
||||
}
|
||||
{
|
||||
# Grafana's forwarder is a second client beside its browser login, which
|
||||
# stays an interactive client.
|
||||
name = "the CI runner, Grafana and log store forwarder clients are registered as machine clients with their own audience and a signed access token";
|
||||
ok =
|
||||
let
|
||||
h = shared.autheliaOnly;
|
||||
clients = h.services.hyperhive.swarm.authelia.oidc.clients;
|
||||
find = id: lib.findFirst (c: c.id == id) null clients;
|
||||
ok1 =
|
||||
svc:
|
||||
let
|
||||
id = ownClientIdOf h svc;
|
||||
c = find id;
|
||||
in
|
||||
c != null
|
||||
&& c.audience == [ id ]
|
||||
&& c.accessTokenSignedResponseAlg == "RS256"
|
||||
&& c.kind == "machine";
|
||||
browser = find h.services.hyperhive.swarm.grafana.oidc.clientId;
|
||||
in
|
||||
lib.all ok1 ownClientForwarders
|
||||
&&
|
||||
h.services.hyperhive.swarm.grafana.otel.clientId != h.services.hyperhive.swarm.grafana.oidc.clientId
|
||||
&& browser != null
|
||||
&& browser.kind != "machine";
|
||||
}
|
||||
]
|
||||
++ forwarderCases;
|
||||
++ forwarderCases
|
||||
++ hostNetnsForwarderCases;
|
||||
in
|
||||
runGroup "nats-authelia" cases
|
||||
|
|
|
|||
|
|
@ -216,6 +216,24 @@ let
|
|||
secretPublisherHere.systemd.services.swarm-secret-publish.script
|
||||
);
|
||||
}
|
||||
{
|
||||
# The forwarders in the CI runner's, Grafana's and the log store's
|
||||
# containers read their secrets from these paths.
|
||||
name = "the publisher carries the CI runner, Grafana and log store forwarders' secrets";
|
||||
ok =
|
||||
lib.all
|
||||
(
|
||||
id:
|
||||
lib.hasInfix "secret/swarm/services/${id}/oidc/client" (
|
||||
secretPublisherHere.systemd.services.swarm-secret-publish.script
|
||||
)
|
||||
)
|
||||
[
|
||||
"swarm-ci-runner-collector"
|
||||
"swarm-grafana-collector"
|
||||
"swarm-victorialogs-collector"
|
||||
];
|
||||
}
|
||||
{
|
||||
# The review classifier reads its queue client secret from the store
|
||||
# inside its container; authelia registers it on every IdP host.
|
||||
|
|
|
|||
Loading…
Reference in a new issue