swarm-ci-runner, swarm-grafana, swarm-victorialogs: forward each container's journal to the swarm collector
Each container now runs the journal forwarder the queue's and the builder's containers run, pushing to a receiver of its own on the swarm collector. Each authenticates as a new machine OIDC client of its own (swarm.<service>.otel.clientId), registered by a glue file wherever authelia runs, minted by authelia and published to the store by the secret publisher. Grafana's browser login client is unchanged. The fetch unit inside each container reads the client secret with the store identity the host already holds for that service (the runner's, Grafana's, and the log store responder's), now also handed into the container as an nspawn system credential. Each of those three policies gains one read stanza on its forwarder's client path. The host units that use those identities are unchanged. The three containers drop --link-journal=host, and the host collector stops reading them.
This commit is contained in:
atlas
mara
parent
c5dbab3277
commit
c6ca9ab845
20 changed files with 810 additions and 107 deletions
|
|
@ -80,6 +80,35 @@ let
|
|||
# Shared host netns, like every sibling swarm container: Grafana reaches
|
||||
# this at 127.0.0.1:<port>.
|
||||
privateNetwork = false;
|
||||
|
||||
# The container's journal forwarder pushes to its own receiver on the swarm
|
||||
# collector, the route `serviceProducers.swarm-victorialogs` in
|
||||
# ./swarm-otel-service.nix declares. The port suffix is elided on 443, as in
|
||||
# ./swarm-bao.nix.
|
||||
forwarderClientId = cfg.otel.clientId;
|
||||
gatewayPortSuffix = lib.optionalString (
|
||||
gatewayCfg.httpsPort != 443
|
||||
) ":${toString gatewayCfg.httpsPort}";
|
||||
forwarderEndpoint = "https://${hyperhiveCfg.swarm.otel.domain}${gatewayPortSuffix}/${cfg.machine}";
|
||||
forwarderUnit = "swarm-journal-forwarder-oidc";
|
||||
# In the host netns: distinct from every other collector's there, which the
|
||||
# assertion below checks.
|
||||
forwarderTelemetryPort = 8897;
|
||||
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
|
||||
# The responder's store identity, handed into the container as system
|
||||
# credentials for the forwarder's fetch unit. The host units keep reading
|
||||
# the same files.
|
||||
forwarderStoreFiles = lib.filterAttrs (_: source: source != null) logsAuthStoreFiles;
|
||||
# nixos-container expands `extraFlags` unquoted, so a source path must be
|
||||
# one word.
|
||||
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
|
||||
|
||||
# The forwarder reaches the swarm collector and authelia by name over TLS; in
|
||||
# self-signed mode their leaves chain to the hive CA.
|
||||
caTrust = import ./lib/hive-ca-trust.nix {
|
||||
inherit lib gatewayCfg;
|
||||
tlsCfg = deployCfg.hive-controller.tls;
|
||||
};
|
||||
in
|
||||
{
|
||||
# What the store IS from any hive's point of view, the name it answers on and
|
||||
|
|
@ -211,6 +240,51 @@ in
|
|||
both options.
|
||||
'';
|
||||
}
|
||||
{
|
||||
assertion = lib.all oneWord (lib.attrValues forwarderStoreFiles);
|
||||
message = ''
|
||||
The log store responder's store identity paths must not contain
|
||||
whitespace: nspawn's --load-credential receives them through
|
||||
nixos-container's unquoted extra flags.
|
||||
'';
|
||||
}
|
||||
{
|
||||
assertion =
|
||||
(hyperhiveCfg.swarm.otel.serviceProducers.${cfg.machine}.clientId or null) == forwarderClientId;
|
||||
message = ''
|
||||
services.hyperhive.swarm.otel.serviceProducers has no entry
|
||||
"${cfg.machine}" with clientId "${forwarderClientId}", so the swarm
|
||||
collector has no receiver for the log store's journal forwarder.
|
||||
Declare it in swarm-otel-service.nix.
|
||||
'';
|
||||
}
|
||||
{
|
||||
assertion =
|
||||
!(lib.elem forwarderTelemetryPort [
|
||||
hyperhiveCfg.otel.telemetryPort
|
||||
hyperhiveCfg.swarm.otel.telemetryPort
|
||||
hyperhiveCfg.swarm.bao.otel.telemetryPort
|
||||
hyperhiveCfg.swarm.nats.otel.telemetryPort
|
||||
]);
|
||||
message = ''
|
||||
The log store's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
|
||||
collides with another collector's in the host netns: the hive
|
||||
collector's, the swarm collector's, the store forwarder's or the
|
||||
queue forwarder's.
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
# nspawn fails to start the container when a credential source is
|
||||
# missing, so where this host mints the leaf the container pulls the
|
||||
# minting unit in and waits for it, as ./swarm-builder.nix's does. The CA
|
||||
# ordering keeps the trust bundle's bind source there first.
|
||||
systemd.services."container@${cfg.machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
{
|
||||
after = [ "swarm-bao-pki.service" ];
|
||||
wants = [ "swarm-bao-pki.service" ];
|
||||
}
|
||||
];
|
||||
|
||||
# Which check a `/select/logsql/` request gets. `auth_request` takes a
|
||||
|
|
@ -506,24 +580,62 @@ in
|
|||
containers.${cfg.machine} = {
|
||||
autoStart = true;
|
||||
ephemeral = false;
|
||||
# Journal files on the host, not inside the container: nixpkgs hardcodes
|
||||
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
|
||||
extraFlags = [ "--link-journal=host" ];
|
||||
# No `--link-journal=host`: the journal stays in the guest, where the
|
||||
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
|
||||
extraFlags = lib.mapAttrsToList (
|
||||
id: source: "--load-credential=${id}:${source}"
|
||||
) forwarderStoreFiles;
|
||||
inherit privateNetwork;
|
||||
bindMounts = caTrust.bindMount;
|
||||
|
||||
config =
|
||||
{ ... }:
|
||||
{
|
||||
imports = [
|
||||
../container-modules/swarm-container.nix
|
||||
../container-modules/swarm-journal-forwarder.nix
|
||||
(import ../container-modules/swarm-container-resolver.nix {
|
||||
inherit (networkCfg) bridgeIp;
|
||||
dnsConsumers = [ "victorialogs.service" ];
|
||||
# The forwarder and its secret fetch dial the swarm collector,
|
||||
# authelia and the store by name too.
|
||||
dnsConsumers = [
|
||||
"victorialogs.service"
|
||||
"opentelemetry-collector.service"
|
||||
"${forwarderUnit}.service"
|
||||
];
|
||||
})
|
||||
# Go: `SSL_CERT_FILE` replaces its trust store, for the swarm
|
||||
# collector's and authelia's names.
|
||||
(caTrust.trustBundle {
|
||||
inherit pkgs;
|
||||
name = cfg.machine;
|
||||
consumers = [ "opentelemetry-collector" ];
|
||||
})
|
||||
];
|
||||
|
||||
services.hyperhive.swarmContainer = { inherit privateNetwork; };
|
||||
|
||||
# Authenticated as the forwarder's own OIDC client, with the
|
||||
# responder's store identity: one identity for the whole container.
|
||||
# It is a system credential, so the fetch unit loads it.
|
||||
services.hyperhive.swarmJournalForwarder = {
|
||||
enable = true;
|
||||
serviceName = "victorialogs";
|
||||
endpoint = forwarderEndpoint;
|
||||
clientId = forwarderClientId;
|
||||
tokenUrl = "${toString hyperhiveCfg.swarm.authelia.url}/api/oidc/token";
|
||||
telemetryPort = forwarderTelemetryPort;
|
||||
store = {
|
||||
package = deployCfg.bao.package;
|
||||
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
secretPath = "secret/swarm/services/${forwarderClientId}/oidc/client";
|
||||
clientCertFile = forwarderCredential "bao-client.pem";
|
||||
clientKeyFile = forwarderCredential "bao-client-key.pem";
|
||||
caFile = if forwarderStoreFiles ? "bao-ca.pem" then forwarderCredential "bao-ca.pem" else null;
|
||||
};
|
||||
};
|
||||
systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames forwarderStoreFiles;
|
||||
|
||||
services.victorialogs = {
|
||||
enable = true;
|
||||
package = deployCfg.victorialogs.package;
|
||||
|
|
|
|||
Loading…
Reference in a new issue