swarm-ci-runner, swarm-grafana, swarm-victorialogs: forward each container's journal to the swarm collector
Each container now runs the journal forwarder the queue's and the builder's containers run, pushing to a receiver of its own on the swarm collector. Each authenticates as a new machine OIDC client of its own (swarm.<service>.otel.clientId), registered by a glue file wherever authelia runs, minted by authelia and published to the store by the secret publisher. Grafana's browser login client is unchanged. The fetch unit inside each container reads the client secret with the store identity the host already holds for that service (the runner's, Grafana's, and the log store responder's), now also handed into the container as an nspawn system credential. Each of those three policies gains one read stanza on its forwarder's client path. The host units that use those identities are unchanged. The three containers drop --link-journal=host, and the host collector stops reading them.
This commit is contained in:
atlas
mara
parent
c5dbab3277
commit
c6ca9ab845
20 changed files with 810 additions and 107 deletions
|
|
@ -55,9 +55,40 @@ let
|
|||
consumers = [
|
||||
runnerUnit
|
||||
registerUnit
|
||||
# Go: `SSL_CERT_FILE` replaces its trust store, for the swarm
|
||||
# collector's and authelia's names.
|
||||
"opentelemetry-collector"
|
||||
];
|
||||
};
|
||||
|
||||
# The container's journal forwarder pushes to its own receiver on the swarm
|
||||
# collector, the route `serviceProducers.swarm-ci-runner` in
|
||||
# ./swarm-otel-service.nix declares. The port suffix is elided on 443, as in
|
||||
# ./swarm-bao.nix.
|
||||
swarmOtelCfg = config.services.hyperhive.swarm.otel;
|
||||
forwarderClientId = config.services.hyperhive.swarm.ci-runner.otel.clientId;
|
||||
gatewayPortSuffix = lib.optionalString (
|
||||
gatewayCfg.httpsPort != 443
|
||||
) ":${toString gatewayCfg.httpsPort}";
|
||||
forwarderEndpoint = "https://${swarmOtelCfg.domain}${gatewayPortSuffix}/${machine}";
|
||||
forwarderTokenUrl = "${toString config.services.hyperhive.swarm.authelia.url}/api/oidc/token";
|
||||
forwarderUnit = "swarm-journal-forwarder-oidc";
|
||||
# Loopback in the container's own netns, and distinct from the other service
|
||||
# containers' forwarders all the same.
|
||||
forwarderTelemetryPort = 8895;
|
||||
|
||||
# The runner's store identity, handed into the container as system
|
||||
# credentials (credential id → host source) for the forwarder's fetch unit.
|
||||
storeFiles = lib.filterAttrs (_: source: source != null) {
|
||||
ci-runner-bao-cert = baoDeploy.ciRunnerClientCertFile;
|
||||
ci-runner-bao-key = baoDeploy.ciRunnerClientKeyFile;
|
||||
ci-runner-bao-server-ca = baoDeploy.serverCaFile;
|
||||
};
|
||||
# nixos-container expands `extraFlags` unquoted, so a source path must be
|
||||
# one word.
|
||||
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
|
||||
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
|
||||
|
||||
# The upstream unit is `DynamicUser`, so its state dir is
|
||||
# `/var/lib/private/gitea-runner`; `/var/lib/gitea-runner` is a symlink to
|
||||
# it, which is the spelling a step run as root (`+`) can follow.
|
||||
|
|
@ -99,6 +130,21 @@ let
|
|||
privateNetwork = true;
|
||||
in
|
||||
{
|
||||
options.services.hyperhive.swarm.ci-runner.otel.clientId = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
readOnly = true;
|
||||
default = "swarm-ci-runner-collector";
|
||||
description = ''
|
||||
OAuth2 client id the journal forwarder inside the CI runner's container
|
||||
authenticates as, and the audience it asks its token for.
|
||||
|
||||
Read-only because three hosts have to agree on it: authelia registers
|
||||
the client (`glue-ci-runner-otel-oidc-client.nix`), the swarm collector
|
||||
admits it on the container's receiver (`swarm.otel.serviceProducers`),
|
||||
and the secret publisher carries its secret to the store.
|
||||
'';
|
||||
};
|
||||
|
||||
options.services.hyperhive.deploy.ci-runner = {
|
||||
# `enable` is in ./deploy.nix.
|
||||
|
||||
|
|
@ -178,6 +224,14 @@ in
|
|||
options.
|
||||
'';
|
||||
}
|
||||
{
|
||||
assertion = lib.all oneWord (lib.attrValues storeFiles);
|
||||
message = ''
|
||||
The CI runner's store identity paths must not contain whitespace:
|
||||
nspawn's --load-credential receives them through nixos-container's
|
||||
unquoted extra flags.
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
# The container hangs off the bridge, takes its address from the
|
||||
|
|
@ -274,7 +328,16 @@ in
|
|||
|
||||
systemd.timers.swarm-bao-ci-runner = storeRefreshTimer "re-read the CI runner's registration token from the swarm secret store";
|
||||
|
||||
systemd.services."container@${machine}" = caTrust.containerOrdering;
|
||||
# nspawn fails to start the container when a credential source is
|
||||
# missing, so where this host mints the leaf the container pulls the
|
||||
# minting unit in and waits for it, as ./swarm-builder.nix's does.
|
||||
systemd.services."container@${machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
{
|
||||
after = [ "swarm-bao-pki.service" ];
|
||||
wants = [ "swarm-bao-pki.service" ];
|
||||
}
|
||||
];
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
# The bind source must exist when the container starts.
|
||||
|
|
@ -287,9 +350,9 @@ in
|
|||
containers.${machine} = {
|
||||
autoStart = true;
|
||||
ephemeral = false;
|
||||
# Journal files on the host, not inside the container: nixpkgs hardcodes
|
||||
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
|
||||
extraFlags = [ "--link-journal=host" ];
|
||||
# No `--link-journal=host`: the journal stays in the guest, where the
|
||||
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
|
||||
extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
|
||||
inherit privateNetwork;
|
||||
hostBridge = networkCfg.bridgeName;
|
||||
|
||||
|
|
@ -338,9 +401,35 @@ in
|
|||
{
|
||||
imports = [
|
||||
../container-modules/swarm-container.nix
|
||||
../container-modules/swarm-journal-forwarder.nix
|
||||
caBundleModule
|
||||
];
|
||||
|
||||
# Authenticated as the forwarder's own OIDC client, with the runner's
|
||||
# store identity: one identity for the whole container. It is a
|
||||
# system credential, so the fetch unit loads it.
|
||||
services.hyperhive.swarmJournalForwarder = {
|
||||
enable = true;
|
||||
serviceName = "ci-runner";
|
||||
endpoint = forwarderEndpoint;
|
||||
clientId = forwarderClientId;
|
||||
tokenUrl = forwarderTokenUrl;
|
||||
telemetryPort = forwarderTelemetryPort;
|
||||
store = {
|
||||
package = baoDeploy.package;
|
||||
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
secretPath = "secret/swarm/services/${forwarderClientId}/oidc/client";
|
||||
clientCertFile = forwarderCredential "ci-runner-bao-cert";
|
||||
clientKeyFile = forwarderCredential "ci-runner-bao-key";
|
||||
caFile =
|
||||
if storeFiles ? ci-runner-bao-server-ca then
|
||||
forwarderCredential "ci-runner-bao-server-ca"
|
||||
else
|
||||
null;
|
||||
};
|
||||
};
|
||||
systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames storeFiles;
|
||||
|
||||
# Its own netns, so it keeps its own firewall; resolvconf stays on
|
||||
# and serves `networking.nameservers` below.
|
||||
services.hyperhive.swarmContainer = {
|
||||
|
|
|
|||
Loading…
Reference in a new issue