ship the cache's init units' journal from a host with no collector, with the cache's own client
This commit is contained in:
parent
5e5ff64b3c
commit
a3682a5e9a
5 changed files with 137 additions and 7 deletions
|
|
@ -396,6 +396,8 @@ A host that runs the cache without the swarm collector adds `swarm-bao-attic-sec
|
||||||
and `swarm-bao-attic-tokens.service`; a host that runs the archive adds `swarm-archive.service`, `swarm-archive-users.service`, `swarm-archive-prune.service` and `swarm-archive-check.service`, and a hive adds `hive-archive-backup.service`.
|
and `swarm-bao-attic-tokens.service`; a host that runs the archive adds `swarm-archive.service`, `swarm-archive-users.service`, `swarm-archive-prune.service` and `swarm-archive-check.service`, and a hive adds `hive-archive-backup.service`.
|
||||||
A host that also runs the swarm collector names none of these: its whole-host journal receiver
|
A host that also runs the swarm collector names none of these: its whole-host journal receiver
|
||||||
ships them already, and the hive collector then has no `journald` receiver.
|
ships them already, and the hive collector then has no `journald` receiver.
|
||||||
|
A host that runs the cache and neither collector ships the cache's two units with
|
||||||
|
its own journal forwarder instead (see [the cache's journal](../swarm/builder.md)).
|
||||||
No bind mount is needed. Unlike an agent's journal, this collector already
|
No bind mount is needed. Unlike an agent's journal, this collector already
|
||||||
shares a filesystem with the units it reads, because it's a host systemd
|
shares a filesystem with the units it reads, because it's a host systemd
|
||||||
service rather than a container. It forwards the crash log line alongside the
|
service rather than a container. It forwards the crash log line alongside the
|
||||||
|
|
|
||||||
|
|
@ -751,5 +751,10 @@ The container forwards atticd's journal to the swarm collector like the other
|
||||||
service containers (see [Telemetry collector](telemetry.md#telemetry-collector-otel)). The two host
|
service containers (see [Telemetry collector](telemetry.md#telemetry-collector-otel)). The two host
|
||||||
units' lines reach the log store too: through the hive collector's journal
|
units' lines reach the log store too: through the hive collector's journal
|
||||||
allow-list on a hive host, or through the swarm collector's whole-journal
|
allow-list on a hive host, or through the swarm collector's whole-journal
|
||||||
receiver where the host runs that. A host with neither keeps them in its own
|
receiver where the host runs that. A host with neither runs the container's
|
||||||
journal.
|
forwarder module on the host itself, for those two units only. It logs in with
|
||||||
|
the `attic` leaf, reads the cache's own OIDC client secret and pushes to the
|
||||||
|
cache's route on the swarm collector, so it needs no new client, grant or leaf.
|
||||||
|
The records carry `service.name` `attic-host`. It only ships the cache's two
|
||||||
|
units: another swarm service's host units on such a host stay in its own
|
||||||
|
journal, because each service has its own client.
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,11 @@
|
||||||
# nested inside the host's, and reading the host side back from in here
|
# nested inside the host's, and reading the host side back from in here
|
||||||
# recurses.
|
# recurses.
|
||||||
#
|
#
|
||||||
|
# One host imports it too: a host with neither the hive collector nor the swarm
|
||||||
|
# collector, whose own units would ship nowhere (./swarm-attic.nix). There the
|
||||||
|
# options sit beside the host's own, `units` names what to ship, and the
|
||||||
|
# journal is the host's.
|
||||||
|
#
|
||||||
# The client secret is fetched inside the container, under the container's own
|
# The client secret is fetched inside the container, under the container's own
|
||||||
# store identity, by `swarm-journal-forwarder-oidc`. That needs the container to
|
# store identity, by `swarm-journal-forwarder-oidc`. That needs the container to
|
||||||
# reach the store, authelia and the swarm collector by name: the container needs
|
# reach the store, authelia and the swarm collector by name: the container needs
|
||||||
|
|
@ -64,6 +69,19 @@ in
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
|
units = lib.mkOption {
|
||||||
|
type = lib.types.nullOr (lib.types.nonEmptyListOf lib.types.str);
|
||||||
|
default = null;
|
||||||
|
example = [ "swarm-bao-attic-tokens.service" ];
|
||||||
|
description = ''
|
||||||
|
Only these systemd units' entries are shipped. Null ships the whole
|
||||||
|
journal, which is right in a container that holds one service. Set it
|
||||||
|
when the module runs on a host, which holds everyone's units and may
|
||||||
|
be someone's workstation too. The list is never empty: a journald
|
||||||
|
receiver with no `units` reads every unit, not none.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
endpoint = lib.mkOption {
|
endpoint = lib.mkOption {
|
||||||
type = lib.types.str;
|
type = lib.types.str;
|
||||||
example = "https://otel.swarm.example/swarm-nats";
|
example = "https://otel.swarm.example/swarm-nats";
|
||||||
|
|
@ -226,7 +244,8 @@ in
|
||||||
# entry for the current boot crash-loops the receiver.
|
# entry for the current boot crash-loops the receiver.
|
||||||
merge = true;
|
merge = true;
|
||||||
storage = "file_storage";
|
storage = "file_storage";
|
||||||
};
|
}
|
||||||
|
// lib.optionalAttrs (cfg.units != null) { inherit (cfg) units; };
|
||||||
|
|
||||||
processors.resource.attributes = [
|
processors.resource.attributes = [
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -119,6 +119,18 @@ let
|
||||||
forwarderTelemetryPort = 8903;
|
forwarderTelemetryPort = 8903;
|
||||||
forwarderOtlpPort = 8905;
|
forwarderOtlpPort = 8905;
|
||||||
|
|
||||||
|
# The host's init units' journal. The hive collector ships it where one runs
|
||||||
|
# (`otel.journalUnits`), and the swarm collector reads the whole journal where
|
||||||
|
# that runs. A host with neither ships it with the forwarder module on the
|
||||||
|
# host itself, as the cache's own client, so nothing new is registered. The
|
||||||
|
# loopback port is the host's, apart from the container's own netns.
|
||||||
|
initUnits = [
|
||||||
|
"swarm-bao-attic-secret.service"
|
||||||
|
"swarm-bao-attic-tokens.service"
|
||||||
|
];
|
||||||
|
shipsOwnJournal = !hyperhiveCfg.otel.collector.enable && !hyperhiveCfg.deploy.swarm-otel.enable;
|
||||||
|
hostForwarderTelemetryPort = 8907;
|
||||||
|
|
||||||
# The exporter's unit and query. `V` is attic's stored value for a valid
|
# The exporter's unit and query. `V` is attic's stored value for a valid
|
||||||
# chunk (`ChunkState::Valid`) and for a valid NAR (`NarState::Valid`).
|
# chunk (`ChunkState::Valid`) and for a valid NAR (`NarState::Valid`).
|
||||||
# `file_size` is the compressed size on disk, and each chunk is one row
|
# `file_size` is the compressed size on disk, and each chunk is one row
|
||||||
|
|
@ -169,6 +181,16 @@ in
|
||||||
hostUnit = true;
|
hostUnit = true;
|
||||||
inherit (cfg) enable;
|
inherit (cfg) enable;
|
||||||
})
|
})
|
||||||
|
# The host's own forwarder below dials the swarm collector and authelia by
|
||||||
|
# name, as the container's does.
|
||||||
|
(caTrust.trustBundle {
|
||||||
|
inherit pkgs;
|
||||||
|
name = "swarm-attic-host-journal";
|
||||||
|
consumers = [ "opentelemetry-collector" ];
|
||||||
|
hostUnit = true;
|
||||||
|
enable = cfg.enable && shipsOwnJournal;
|
||||||
|
})
|
||||||
|
../container-modules/swarm-journal-forwarder.nix
|
||||||
];
|
];
|
||||||
|
|
||||||
options.services.hyperhive.deploy.attic = {
|
options.services.hyperhive.deploy.attic = {
|
||||||
|
|
@ -292,10 +314,30 @@ in
|
||||||
# The init units' journal reaches the log store through the hive collector.
|
# The init units' journal reaches the log store through the hive collector.
|
||||||
# A host that also runs the swarm collector ships its whole journal already
|
# A host that also runs the swarm collector ships its whole journal already
|
||||||
# (./swarm-otel.nix), so naming them there would send each line twice.
|
# (./swarm-otel.nix), so naming them there would send each line twice.
|
||||||
services.hyperhive.otel.journalUnits = lib.optionals (!hyperhiveCfg.deploy.swarm-otel.enable) [
|
services.hyperhive.otel.journalUnits = lib.optionals (
|
||||||
"swarm-bao-attic-secret.service"
|
!hyperhiveCfg.deploy.swarm-otel.enable
|
||||||
"swarm-bao-attic-tokens.service"
|
) initUnits;
|
||||||
];
|
|
||||||
|
# A host with no collector of its own: the units' journal goes to the swarm
|
||||||
|
# collector's route for the cache, which the cache's client and leaf
|
||||||
|
# already reach (the container's forwarder below uses both).
|
||||||
|
services.hyperhive.swarmJournalForwarder = lib.mkIf shipsOwnJournal {
|
||||||
|
enable = true;
|
||||||
|
serviceName = "attic-host";
|
||||||
|
units = initUnits;
|
||||||
|
endpoint = forwarderEndpoint;
|
||||||
|
clientId = forwarderClientId;
|
||||||
|
tokenUrl = "${toString autheliaCfg.url}/api/oidc/token";
|
||||||
|
telemetryPort = hostForwarderTelemetryPort;
|
||||||
|
store = {
|
||||||
|
package = baoDeploy.package;
|
||||||
|
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||||
|
secretPath = "secret/swarm/services/${forwarderClientId}/oidc/client";
|
||||||
|
clientCertFile = baoDeploy.atticClientCertFile;
|
||||||
|
clientKeyFile = baoDeploy.atticClientKeyFile;
|
||||||
|
caFile = baoDeploy.serverCaFile;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
# This host serves the vhost, and the host's own resolver finds the name.
|
# This host serves the vhost, and the host's own resolver finds the name.
|
||||||
services.hyperhive.gateway.enable = lib.mkDefault true;
|
services.hyperhive.gateway.enable = lib.mkDefault true;
|
||||||
|
|
|
||||||
|
|
@ -119,6 +119,14 @@ let
|
||||||
prefix = "archive";
|
prefix = "archive";
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
# The cache by hand on a host with neither the hive collector nor the swarm
|
||||||
|
# collector: the one place its init units' journal has no collector to go
|
||||||
|
# through.
|
||||||
|
atticAlone = hive {
|
||||||
|
deploy.bao.enable = true;
|
||||||
|
deploy.attic.enable = true;
|
||||||
|
};
|
||||||
|
forwarderOf = m: m.services.hyperhive.swarmJournalForwarder;
|
||||||
atticWarnings =
|
atticWarnings =
|
||||||
m:
|
m:
|
||||||
lib.length (
|
lib.length (
|
||||||
|
|
@ -494,6 +502,60 @@ let
|
||||||
&& hasNone atticExternal
|
&& hasNone atticExternal
|
||||||
&& lib.elem "hive-c0re.service" (units atticByHand);
|
&& lib.elem "hive-c0re.service" (units atticByHand);
|
||||||
}
|
}
|
||||||
|
{
|
||||||
|
# With neither collector the init units' journal would ship nowhere, so
|
||||||
|
# the host runs the forwarder module itself, for those two units only,
|
||||||
|
# to the cache's own route as the cache's own client. `atticByHand` (hive
|
||||||
|
# collector) and `atticHere` (swarm collector) are the controls: their
|
||||||
|
# journal already goes through a collector, so a second forwarder would
|
||||||
|
# send each line twice.
|
||||||
|
name = "a host with no collector forwards the cache's init units itself, and a host with one does not";
|
||||||
|
ok =
|
||||||
|
let
|
||||||
|
f = forwarderOf atticAlone;
|
||||||
|
settings = atticAlone.services.opentelemetry-collector.settings;
|
||||||
|
svcs = atticAlone.systemd.services;
|
||||||
|
in
|
||||||
|
f.enable
|
||||||
|
&&
|
||||||
|
f.units == [
|
||||||
|
"swarm-bao-attic-secret.service"
|
||||||
|
"swarm-bao-attic-tokens.service"
|
||||||
|
]
|
||||||
|
&& settings.receivers.journald.units == f.units
|
||||||
|
&& settings.receivers.journald.directory == "/var/log/journal"
|
||||||
|
&& f.clientId == "swarm-attic-collector"
|
||||||
|
&& f.endpoint == "https://${atticAlone.services.hyperhive.swarm.otel.domain}/swarm-attic";
|
||||||
|
}
|
||||||
|
{
|
||||||
|
# The control for the case above: the same lookups find the forwarder on
|
||||||
|
# the host that needs it and none where a collector already ships.
|
||||||
|
name = "the host's cache forwarder is absent where the hive or the swarm collector ships the journal";
|
||||||
|
ok =
|
||||||
|
(forwarderOf atticAlone).enable
|
||||||
|
&& !(forwarderOf atticByHand).enable
|
||||||
|
&& !(forwarderOf atticHere).enable
|
||||||
|
&& !(atticByHand.systemd.services ? swarm-journal-forwarder-oidc)
|
||||||
|
&& !(atticHere.systemd.services ? swarm-journal-forwarder-oidc)
|
||||||
|
&& atticAlone.systemd.services ? swarm-journal-forwarder-oidc;
|
||||||
|
}
|
||||||
|
{
|
||||||
|
# The forwarder logs in with the cache's leaf and reads the cache's own
|
||||||
|
# client secret, the path the container's forwarder reads, so no grant and
|
||||||
|
# no client is new. The container's receiver ships its whole journal.
|
||||||
|
name = "the host's cache forwarder uses the cache's leaf and client, and the container still ships its whole journal";
|
||||||
|
ok =
|
||||||
|
let
|
||||||
|
env = atticAlone.systemd.services.swarm-journal-forwarder-oidc.environment;
|
||||||
|
inner = atticAlone.containers.swarm-attic.config.services.opentelemetry-collector.settings;
|
||||||
|
in
|
||||||
|
env.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/attic.pem"
|
||||||
|
&& env.BAO_CLIENT_KEY == "/var/lib/swarm-bao-pki/attic-key.pem"
|
||||||
|
&&
|
||||||
|
(forwarderOf atticAlone).store.secretPath
|
||||||
|
== "secret/swarm/services/swarm-attic-collector/oidc/client"
|
||||||
|
&& !(inner.receivers.journald ? units);
|
||||||
|
}
|
||||||
{
|
{
|
||||||
# The hive collector ships the container units' start output (the rootfs
|
# The hive collector ships the container units' start output (the rootfs
|
||||||
# check's lines) unless the swarm collector already reads the whole
|
# check's lines) unless the swarm collector already reads the whole
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue