forward the attic container's journal to the swarm collector
This commit is contained in:
atlas
parent
99fab06348
commit
a572fb6a7f
11 changed files with 230 additions and 21 deletions
|
|
@ -52,6 +52,7 @@
|
|||
./glue-ci-runner-otel-oidc-client.nix
|
||||
./glue-claude-login-oidc-client.nix
|
||||
./glue-notify-router-oidc-client.nix
|
||||
./glue-attic-otel-oidc-client.nix
|
||||
./glue-forge-otel-oidc-client.nix
|
||||
./glue-grafana-otel-oidc-client.nix
|
||||
./glue-matrix-otel-oidc-client.nix
|
||||
|
|
|
|||
37
nix/host-modules/glue-attic-otel-oidc-client.nix
Normal file
37
nix/host-modules/glue-attic-otel-oidc-client.nix
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
# Glue: register the cache container's journal forwarder as an OIDC client
|
||||
# wherever authelia runs.
|
||||
#
|
||||
# ONE PAIRING PER FILE: that forwarder ← authelia, and nothing else.
|
||||
# Without it authelia mints no secret, the publisher has nothing to copy into
|
||||
# the store, and the forwarder's export is a 401.
|
||||
#
|
||||
# Gated on authelia being HERE, not on this host running the cache: a client is
|
||||
# a row in THIS host's provider config, the split
|
||||
# ./glue-victorialogs-otel-oidc-client.nix makes for the same reason.
|
||||
{
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
let
|
||||
hyperhiveCfg = config.services.hyperhive;
|
||||
clientId = hyperhiveCfg.swarm.attic.otel.clientId;
|
||||
in
|
||||
{
|
||||
config = lib.mkIf hyperhiveCfg.deploy.authelia.enable {
|
||||
services.hyperhive.swarm.authelia.oidc.clients = [
|
||||
{
|
||||
id = clientId;
|
||||
description = "HyperHive cache journal forwarder";
|
||||
kind = "machine";
|
||||
redirectUris = [ ];
|
||||
# Its own id as its own permitted audience, the one the swarm
|
||||
# collector's receiver for this container checks.
|
||||
audience = [ clientId ];
|
||||
# The receiver verifies the token offline against `/jwks.json`;
|
||||
# authelia's default opaque handle fails there.
|
||||
accessTokenSignedResponseAlg = "RS256";
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
|
|
@ -13,6 +13,21 @@ let
|
|||
in
|
||||
{
|
||||
options.services.hyperhive.swarm.attic = {
|
||||
otel.clientId = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
readOnly = true;
|
||||
default = "swarm-attic-collector";
|
||||
description = ''
|
||||
OAuth2 client id the journal forwarder inside the cache's container
|
||||
authenticates as, and the audience it asks its token for.
|
||||
|
||||
Read-only because three hosts have to agree on it: authelia registers
|
||||
the client (`glue-attic-otel-oidc-client.nix`), the swarm collector
|
||||
admits it on the container's receiver (`swarm.otel.serviceProducers`),
|
||||
and the secret publisher carries its secret to the store.
|
||||
'';
|
||||
};
|
||||
|
||||
ownDomain = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
readOnly = true;
|
||||
|
|
|
|||
|
|
@ -29,6 +29,9 @@ let
|
|||
atticCfg = hyperhiveCfg.swarm.attic;
|
||||
baoCfg = hyperhiveCfg.swarm.bao;
|
||||
baoDeploy = hyperhiveCfg.deploy.bao;
|
||||
autheliaCfg = hyperhiveCfg.swarm.authelia;
|
||||
networkCfg = hyperhiveCfg.network;
|
||||
gatewayCfg = hyperhiveCfg.gateway;
|
||||
|
||||
machine = "swarm-attic";
|
||||
unit = "swarm-attic";
|
||||
|
|
@ -87,6 +90,33 @@ let
|
|||
|
||||
haveClientIdentity = baoDeploy.atticClientCertFile != null && baoDeploy.atticClientKeyFile != null;
|
||||
|
||||
# The container's journal forwarder pushes to its own receiver on the swarm
|
||||
# collector, the route `serviceProducers.swarm-attic` in
|
||||
# ./swarm-otel-service.nix declares.
|
||||
forwarderClientId = atticCfg.otel.clientId;
|
||||
forwarderEndpoint = "https://${hyperhiveCfg.swarm.otel.domain}/${machine}";
|
||||
forwarderUnit = "swarm-journal-forwarder-oidc";
|
||||
# In the host netns: distinct from every other collector's there.
|
||||
forwarderTelemetryPort = 8903;
|
||||
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
|
||||
# The cache's store identity, handed into the container as system credentials
|
||||
# for the forwarder's fetch unit. The host units keep reading the same files.
|
||||
forwarderStoreFiles = lib.filterAttrs (_: source: source != null) {
|
||||
attic-bao-cert = baoDeploy.atticClientCertFile;
|
||||
attic-bao-key = baoDeploy.atticClientKeyFile;
|
||||
attic-bao-server-ca = baoDeploy.serverCaFile;
|
||||
};
|
||||
# nixos-container expands `extraFlags` unquoted, so a source path must be
|
||||
# one word.
|
||||
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
|
||||
|
||||
# The forwarder reaches the swarm collector and authelia by name over TLS; in
|
||||
# self-signed mode their leaves chain to the hive CA.
|
||||
caTrust = import ./lib/hive-ca-trust.nix {
|
||||
inherit lib gatewayCfg;
|
||||
tlsCfg = hyperhiveCfg.deploy.hive-controller.tls;
|
||||
};
|
||||
|
||||
# Both host units log in as the cache's principal.
|
||||
storeEnvironment = {
|
||||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
|
|
@ -171,6 +201,35 @@ in
|
|||
is set to null.
|
||||
'';
|
||||
}
|
||||
{
|
||||
assertion = lib.all oneWord (lib.attrValues forwarderStoreFiles);
|
||||
message = ''
|
||||
The cache's store identity paths must not contain whitespace:
|
||||
nspawn's --load-credential receives them through nixos-container's
|
||||
unquoted extra flags.
|
||||
'';
|
||||
}
|
||||
{
|
||||
assertion =
|
||||
(hyperhiveCfg.swarm.otel.serviceProducers.${machine}.clientId or null) == forwarderClientId;
|
||||
message = ''
|
||||
services.hyperhive.swarm.otel.serviceProducers has no entry
|
||||
"${machine}" with clientId "${forwarderClientId}", so the swarm
|
||||
collector has no receiver for the cache's journal forwarder.
|
||||
Declare it in swarm-otel-service.nix.
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
# nspawn fails to start the container when a credential source is
|
||||
# missing, so where this host mints the leaf the container waits for it. The
|
||||
# CA ordering keeps the trust bundle's bind source there first.
|
||||
systemd.services."container@${machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
{
|
||||
after = [ "swarm-bao-pki.service" ];
|
||||
wants = [ "swarm-bao-pki.service" ];
|
||||
}
|
||||
];
|
||||
|
||||
services.hyperhive.gateway.localNames = [ atticCfg.domain ];
|
||||
|
|
@ -279,19 +338,71 @@ in
|
|||
containers.${machine} = {
|
||||
autoStart = true;
|
||||
ephemeral = false;
|
||||
# No `--link-journal=host`: the journal stays in the guest, where the
|
||||
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
|
||||
extraFlags = lib.mapAttrsToList (
|
||||
id: source: "--load-credential=${id}:${source}"
|
||||
) forwarderStoreFiles;
|
||||
privateNetwork = false;
|
||||
bindMounts.${secretDir} = {
|
||||
hostPath = secretDir;
|
||||
isReadOnly = true;
|
||||
};
|
||||
bindMounts = {
|
||||
${secretDir} = {
|
||||
hostPath = secretDir;
|
||||
isReadOnly = true;
|
||||
};
|
||||
}
|
||||
// caTrust.bindMount;
|
||||
|
||||
config =
|
||||
{ ... }:
|
||||
{
|
||||
imports = [ ../container-modules/swarm-container.nix ];
|
||||
imports = [
|
||||
../container-modules/swarm-container.nix
|
||||
../container-modules/swarm-journal-forwarder.nix
|
||||
(import ../container-modules/swarm-container-resolver.nix {
|
||||
inherit (networkCfg) bridgeIp;
|
||||
# The forwarder and its secret fetch dial the swarm collector,
|
||||
# authelia and the store by name.
|
||||
dnsConsumers = [
|
||||
"opentelemetry-collector.service"
|
||||
"${forwarderUnit}.service"
|
||||
];
|
||||
})
|
||||
# Go: `SSL_CERT_FILE` replaces its trust store, for the swarm
|
||||
# collector's and authelia's names.
|
||||
(caTrust.trustBundle {
|
||||
inherit pkgs;
|
||||
name = machine;
|
||||
consumers = [ "opentelemetry-collector" ];
|
||||
})
|
||||
];
|
||||
|
||||
services.hyperhive.swarmContainer.privateNetwork = false;
|
||||
|
||||
# Authenticated as the forwarder's own OIDC client, with the cache's
|
||||
# store identity: one identity for the whole container. It is a
|
||||
# system credential, so the fetch unit loads it.
|
||||
services.hyperhive.swarmJournalForwarder = {
|
||||
enable = true;
|
||||
serviceName = "attic";
|
||||
endpoint = forwarderEndpoint;
|
||||
clientId = forwarderClientId;
|
||||
tokenUrl = "${toString autheliaCfg.url}/api/oidc/token";
|
||||
telemetryPort = forwarderTelemetryPort;
|
||||
store = {
|
||||
package = baoDeploy.package;
|
||||
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
secretPath = "secret/swarm/services/${forwarderClientId}/oidc/client";
|
||||
clientCertFile = forwarderCredential "attic-bao-cert";
|
||||
clientKeyFile = forwarderCredential "attic-bao-key";
|
||||
caFile =
|
||||
if forwarderStoreFiles ? attic-bao-server-ca then
|
||||
forwarderCredential "attic-bao-server-ca"
|
||||
else
|
||||
null;
|
||||
};
|
||||
};
|
||||
systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames forwarderStoreFiles;
|
||||
|
||||
systemd.services.${unit} = {
|
||||
description = "attic binary cache server";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
|
|
|
|||
|
|
@ -1187,7 +1187,9 @@ let
|
|||
|
||||
# The in-swarm cache's three secrets, create and read and nothing more: its
|
||||
# host units create each one only while it is absent, so none needs `update`,
|
||||
# and a rewrite of a signing key or a token an operator holds is refused.
|
||||
# and a rewrite of a signing key or a token an operator holds is refused. The
|
||||
# cache container's journal forwarder reads its client secret under the same
|
||||
# leaf.
|
||||
atticWriters = [
|
||||
{
|
||||
name = "swarm-attic";
|
||||
|
|
@ -1203,7 +1205,8 @@ let
|
|||
atticJwtLeaf
|
||||
atticPushTokenLeaf
|
||||
atticPullTokenLeaf
|
||||
];
|
||||
]
|
||||
+ readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.attic.otel.clientId}/oidc/client";
|
||||
}
|
||||
];
|
||||
|
||||
|
|
|
|||
|
|
@ -304,8 +304,12 @@ in
|
|||
swarm-notify-router = {
|
||||
clientId = hyperhiveCfg.swarm.notify-router.clientId;
|
||||
};
|
||||
# The forwarders in ./swarm-ci-runner.nix, ./swarm-grafana.nix and
|
||||
# ./swarm-victorialogs.nix, each authenticated as a client of its own.
|
||||
# The forwarders in ./swarm-attic.nix, ./swarm-ci-runner.nix,
|
||||
# ./swarm-grafana.nix and ./swarm-victorialogs.nix, each authenticated as a
|
||||
# client of its own.
|
||||
swarm-attic = {
|
||||
clientId = hyperhiveCfg.swarm.attic.otel.clientId;
|
||||
};
|
||||
swarm-ci-runner = {
|
||||
clientId = hyperhiveCfg.swarm.ci-runner.otel.clientId;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -202,7 +202,7 @@ let
|
|||
# `swarm-builder`, `swarm-claude-login`, `swarm-notify-router`,
|
||||
# `swarm-review-classifier`, `swarm-ci-runner`, `swarm-grafana`,
|
||||
# `swarm-victorialogs`, `hive-forge`, `hive-matrix`, `swarm-victoriametrics`,
|
||||
# `swarm-authelia`) forwards its own
|
||||
# `swarm-authelia`, `swarm-attic`) forwards its own
|
||||
# journal from its own collector, so this receiver never sees their units.
|
||||
hostJournalDir = "/var/log/journal";
|
||||
hostJournalMount = "/run/host-journal";
|
||||
|
|
|
|||
|
|
@ -55,8 +55,9 @@ let
|
|||
# The notification router, which reads it itself inside its container.
|
||||
hyperhiveCfg.swarm.notify-router.clientId
|
||||
# The journal forwarders in the CI runner's, Grafana's, the log store's,
|
||||
# the forge's and the homeserver's containers, each reading it inside its
|
||||
# container.
|
||||
# the cache's, the forge's and the homeserver's containers, each reading it
|
||||
# inside its container.
|
||||
hyperhiveCfg.swarm.attic.otel.clientId
|
||||
hyperhiveCfg.swarm.ci-runner.otel.clientId
|
||||
hyperhiveCfg.swarm.grafana.otel.clientId
|
||||
hyperhiveCfg.swarm.victorialogs.otel.clientId
|
||||
|
|
|
|||
|
|
@ -1980,9 +1980,10 @@ let
|
|||
}
|
||||
{
|
||||
# The cache's host units create each of its three secrets and read them
|
||||
# back, and may not replace one: no `update`. The notification router's
|
||||
# grant is the control that `grantsIn` reads a one-path grant too.
|
||||
name = "the cache's grant is create and read on its signing key and its two tokens, and nothing else";
|
||||
# back, and may not replace one: no `update`. Its container's journal
|
||||
# forwarder reads its own client secret, read only. The notification
|
||||
# router's grant is the control that `grantsIn` reads a one-path grant too.
|
||||
name = "the cache's grant is create and read on its signing key and its two tokens, read on its forwarder client, and nothing else";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-attic-policy.script;
|
||||
|
|
@ -1995,7 +1996,9 @@ let
|
|||
(grantFor grants "secret/data/swarm/services/attic/jwt-secret").caps == caps
|
||||
&& (grantFor grants "secret/data/swarm/services/attic/push-token").caps == caps
|
||||
&& (grantFor grants "secret/data/swarm/services/attic/pull-token").caps == caps
|
||||
&& lib.length grants == 3
|
||||
&&
|
||||
(grantFor grants "secret/data/swarm/services/swarm-attic-collector/oidc/client").caps == [ "read" ]
|
||||
&& lib.length grants == 4
|
||||
&& lib.length (grantsIn baoGrantHere.systemd.services.swarm-bao-notify-router-policy.script) == 1
|
||||
&& lib.hasInfix "auth/cert/certs/swarm-attic" s
|
||||
&& lib.hasInfix "allowed_common_names=swarm-attic" s
|
||||
|
|
|
|||
|
|
@ -385,6 +385,36 @@ let
|
|||
&& lib.hasInfix "--mode monolithic" atticUnit.serviceConfig.ExecStart
|
||||
&& atticDeploy.retentionPeriod == "14 days";
|
||||
}
|
||||
{
|
||||
# atticd's journal reaches the swarm collector through the container's own
|
||||
# forwarder, under the cache's store leaf. The host with the switch forced
|
||||
# off is the control that nothing of it renders there, and `atticHere`
|
||||
# that the same lookups find it where the cache runs.
|
||||
name = "the cache's container forwards its journal to the swarm collector as its own client, and only where the cache runs";
|
||||
ok =
|
||||
let
|
||||
inner = atticHere.containers.swarm-attic;
|
||||
f = inner.config.services.hyperhive.swarmJournalForwarder;
|
||||
fetch = inner.config.systemd.services.swarm-journal-forwarder-oidc;
|
||||
collector = inner.config.systemd.services.opentelemetry-collector;
|
||||
in
|
||||
f.enable
|
||||
&& f.serviceName == "attic"
|
||||
&& f.clientId == "swarm-attic-collector"
|
||||
&& f.endpoint == "https://${atticHere.services.hyperhive.swarm.otel.domain}/swarm-attic"
|
||||
&& f.store.secretPath == "secret/swarm/services/swarm-attic-collector/oidc/client"
|
||||
&& f.store.clientCertFile == "/run/credentials/swarm-journal-forwarder-oidc.service/attic-bao-cert"
|
||||
&& lib.elem "attic-bao-key" fetch.serviceConfig.LoadCredential
|
||||
&& lib.elem "--load-credential=attic-bao-cert:/var/lib/swarm-bao-pki/attic.pem" inner.extraFlags
|
||||
&& !(lib.elem "--link-journal=host" inner.extraFlags)
|
||||
&& collector.enable
|
||||
&&
|
||||
atticHere.services.hyperhive.swarm.otel.serviceProducers.swarm-attic.clientId
|
||||
== "swarm-attic-collector"
|
||||
&& lib.hasInfix "services/swarm-attic-collector/oidc/client" atticHere.systemd.services.swarm-bao-attic-policy.script
|
||||
&& !(atticExternal.containers ? swarm-attic)
|
||||
&& atticExternal.services.hyperhive.swarm.otel.serviceProducers ? swarm-attic;
|
||||
}
|
||||
{
|
||||
# attic checks its own tokens and a nix client cannot follow a login
|
||||
# redirect, so the vhost carries no `auth_request`; the tokens ride in a
|
||||
|
|
|
|||
|
|
@ -221,7 +221,7 @@ let
|
|||
# The forwarders in the CI runner's, Grafana's, the log store's, the
|
||||
# forge's, the homeserver's and authelia's containers read their secrets
|
||||
# from these paths, and the metrics store's host reads its container's.
|
||||
name = "the publisher carries the CI runner, Grafana, log store, forge, homeserver, metrics store and authelia forwarders' secrets";
|
||||
name = "the publisher carries the CI runner, Grafana, log store, cache, forge, homeserver, metrics store and authelia forwarders' secrets";
|
||||
ok =
|
||||
lib.all
|
||||
(
|
||||
|
|
@ -238,6 +238,7 @@ let
|
|||
"swarm-matrix-collector"
|
||||
"swarm-victoriametrics-collector"
|
||||
"swarm-authelia-collector"
|
||||
"swarm-attic-collector"
|
||||
];
|
||||
}
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in a new issue