From a3682a5e9a4f1bea78f1e10270e1a1b86f80e3d1 Mon Sep 17 00:00:00 2001 From: maia Date: Wed, 18 Nov 2026 03:38:07 +0100 Subject: [PATCH] ship the cache's init units' journal from a host with no collector, with the cache's own client --- .../scheduler/observability.md | 2 + docs/4-implementation/swarm/builder.md | 9 ++- .../swarm-journal-forwarder.nix | 21 ++++++- nix/host-modules/swarm-attic.nix | 50 +++++++++++++-- nix/module-eval/builder.nix | 62 +++++++++++++++++++ 5 files changed, 137 insertions(+), 7 deletions(-) diff --git a/docs/4-implementation/scheduler/observability.md b/docs/4-implementation/scheduler/observability.md index b34c2c7cb..28afaa71c 100644 --- a/docs/4-implementation/scheduler/observability.md +++ b/docs/4-implementation/scheduler/observability.md @@ -396,6 +396,8 @@ A host that runs the cache without the swarm collector adds `swarm-bao-attic-sec and `swarm-bao-attic-tokens.service`; a host that runs the archive adds `swarm-archive.service`, `swarm-archive-users.service`, `swarm-archive-prune.service` and `swarm-archive-check.service`, and a hive adds `hive-archive-backup.service`. A host that also runs the swarm collector names none of these: its whole-host journal receiver ships them already, and the hive collector then has no `journald` receiver. +A host that runs the cache and neither collector ships the cache's two units with +its own journal forwarder instead (see [the cache's journal](../swarm/builder.md)). No bind mount is needed. Unlike an agent's journal, this collector already shares a filesystem with the units it reads, because it's a host systemd service rather than a container. It forwards the crash log line alongside the diff --git a/docs/4-implementation/swarm/builder.md b/docs/4-implementation/swarm/builder.md index 968d84e90..ff8e41827 100644 --- a/docs/4-implementation/swarm/builder.md +++ b/docs/4-implementation/swarm/builder.md @@ -751,5 +751,10 @@ The container forwards atticd's journal to the swarm collector like the other service containers (see [Telemetry collector](telemetry.md#telemetry-collector-otel)). The two host units' lines reach the log store too: through the hive collector's journal allow-list on a hive host, or through the swarm collector's whole-journal -receiver where the host runs that. A host with neither keeps them in its own -journal. +receiver where the host runs that. A host with neither runs the container's +forwarder module on the host itself, for those two units only. It logs in with +the `attic` leaf, reads the cache's own OIDC client secret and pushes to the +cache's route on the swarm collector, so it needs no new client, grant or leaf. +The records carry `service.name` `attic-host`. It only ships the cache's two +units: another swarm service's host units on such a host stay in its own +journal, because each service has its own client. diff --git a/nix/container-modules/swarm-journal-forwarder.nix b/nix/container-modules/swarm-journal-forwarder.nix index 35240a862..a658f89af 100644 --- a/nix/container-modules/swarm-journal-forwarder.nix +++ b/nix/container-modules/swarm-journal-forwarder.nix @@ -7,6 +7,11 @@ # nested inside the host's, and reading the host side back from in here # recurses. # +# One host imports it too: a host with neither the hive collector nor the swarm +# collector, whose own units would ship nowhere (./swarm-attic.nix). There the +# options sit beside the host's own, `units` names what to ship, and the +# journal is the host's. +# # The client secret is fetched inside the container, under the container's own # store identity, by `swarm-journal-forwarder-oidc`. That needs the container to # reach the store, authelia and the swarm collector by name: the container needs @@ -64,6 +69,19 @@ in ''; }; + units = lib.mkOption { + type = lib.types.nullOr (lib.types.nonEmptyListOf lib.types.str); + default = null; + example = [ "swarm-bao-attic-tokens.service" ]; + description = '' + Only these systemd units' entries are shipped. Null ships the whole + journal, which is right in a container that holds one service. Set it + when the module runs on a host, which holds everyone's units and may + be someone's workstation too. The list is never empty: a journald + receiver with no `units` reads every unit, not none. + ''; + }; + endpoint = lib.mkOption { type = lib.types.str; example = "https://otel.swarm.example/swarm-nats"; @@ -226,7 +244,8 @@ in # entry for the current boot crash-loops the receiver. merge = true; storage = "file_storage"; - }; + } + // lib.optionalAttrs (cfg.units != null) { inherit (cfg) units; }; processors.resource.attributes = [ { diff --git a/nix/host-modules/swarm-attic.nix b/nix/host-modules/swarm-attic.nix index 5bd310a8d..7c1720197 100644 --- a/nix/host-modules/swarm-attic.nix +++ b/nix/host-modules/swarm-attic.nix @@ -119,6 +119,18 @@ let forwarderTelemetryPort = 8903; forwarderOtlpPort = 8905; + # The host's init units' journal. The hive collector ships it where one runs + # (`otel.journalUnits`), and the swarm collector reads the whole journal where + # that runs. A host with neither ships it with the forwarder module on the + # host itself, as the cache's own client, so nothing new is registered. The + # loopback port is the host's, apart from the container's own netns. + initUnits = [ + "swarm-bao-attic-secret.service" + "swarm-bao-attic-tokens.service" + ]; + shipsOwnJournal = !hyperhiveCfg.otel.collector.enable && !hyperhiveCfg.deploy.swarm-otel.enable; + hostForwarderTelemetryPort = 8907; + # The exporter's unit and query. `V` is attic's stored value for a valid # chunk (`ChunkState::Valid`) and for a valid NAR (`NarState::Valid`). # `file_size` is the compressed size on disk, and each chunk is one row @@ -169,6 +181,16 @@ in hostUnit = true; inherit (cfg) enable; }) + # The host's own forwarder below dials the swarm collector and authelia by + # name, as the container's does. + (caTrust.trustBundle { + inherit pkgs; + name = "swarm-attic-host-journal"; + consumers = [ "opentelemetry-collector" ]; + hostUnit = true; + enable = cfg.enable && shipsOwnJournal; + }) + ../container-modules/swarm-journal-forwarder.nix ]; options.services.hyperhive.deploy.attic = { @@ -292,10 +314,30 @@ in # The init units' journal reaches the log store through the hive collector. # A host that also runs the swarm collector ships its whole journal already # (./swarm-otel.nix), so naming them there would send each line twice. - services.hyperhive.otel.journalUnits = lib.optionals (!hyperhiveCfg.deploy.swarm-otel.enable) [ - "swarm-bao-attic-secret.service" - "swarm-bao-attic-tokens.service" - ]; + services.hyperhive.otel.journalUnits = lib.optionals ( + !hyperhiveCfg.deploy.swarm-otel.enable + ) initUnits; + + # A host with no collector of its own: the units' journal goes to the swarm + # collector's route for the cache, which the cache's client and leaf + # already reach (the container's forwarder below uses both). + services.hyperhive.swarmJournalForwarder = lib.mkIf shipsOwnJournal { + enable = true; + serviceName = "attic-host"; + units = initUnits; + endpoint = forwarderEndpoint; + clientId = forwarderClientId; + tokenUrl = "${toString autheliaCfg.url}/api/oidc/token"; + telemetryPort = hostForwarderTelemetryPort; + store = { + package = baoDeploy.package; + addr = "https://${baoCfg.domain}:${toString baoCfg.port}"; + secretPath = "secret/swarm/services/${forwarderClientId}/oidc/client"; + clientCertFile = baoDeploy.atticClientCertFile; + clientKeyFile = baoDeploy.atticClientKeyFile; + caFile = baoDeploy.serverCaFile; + }; + }; # This host serves the vhost, and the host's own resolver finds the name. services.hyperhive.gateway.enable = lib.mkDefault true; diff --git a/nix/module-eval/builder.nix b/nix/module-eval/builder.nix index 30af736cc..cfc39c481 100644 --- a/nix/module-eval/builder.nix +++ b/nix/module-eval/builder.nix @@ -119,6 +119,14 @@ let prefix = "archive"; }; }; + # The cache by hand on a host with neither the hive collector nor the swarm + # collector: the one place its init units' journal has no collector to go + # through. + atticAlone = hive { + deploy.bao.enable = true; + deploy.attic.enable = true; + }; + forwarderOf = m: m.services.hyperhive.swarmJournalForwarder; atticWarnings = m: lib.length ( @@ -494,6 +502,60 @@ let && hasNone atticExternal && lib.elem "hive-c0re.service" (units atticByHand); } + { + # With neither collector the init units' journal would ship nowhere, so + # the host runs the forwarder module itself, for those two units only, + # to the cache's own route as the cache's own client. `atticByHand` (hive + # collector) and `atticHere` (swarm collector) are the controls: their + # journal already goes through a collector, so a second forwarder would + # send each line twice. + name = "a host with no collector forwards the cache's init units itself, and a host with one does not"; + ok = + let + f = forwarderOf atticAlone; + settings = atticAlone.services.opentelemetry-collector.settings; + svcs = atticAlone.systemd.services; + in + f.enable + && + f.units == [ + "swarm-bao-attic-secret.service" + "swarm-bao-attic-tokens.service" + ] + && settings.receivers.journald.units == f.units + && settings.receivers.journald.directory == "/var/log/journal" + && f.clientId == "swarm-attic-collector" + && f.endpoint == "https://${atticAlone.services.hyperhive.swarm.otel.domain}/swarm-attic"; + } + { + # The control for the case above: the same lookups find the forwarder on + # the host that needs it and none where a collector already ships. + name = "the host's cache forwarder is absent where the hive or the swarm collector ships the journal"; + ok = + (forwarderOf atticAlone).enable + && !(forwarderOf atticByHand).enable + && !(forwarderOf atticHere).enable + && !(atticByHand.systemd.services ? swarm-journal-forwarder-oidc) + && !(atticHere.systemd.services ? swarm-journal-forwarder-oidc) + && atticAlone.systemd.services ? swarm-journal-forwarder-oidc; + } + { + # The forwarder logs in with the cache's leaf and reads the cache's own + # client secret, the path the container's forwarder reads, so no grant and + # no client is new. The container's receiver ships its whole journal. + name = "the host's cache forwarder uses the cache's leaf and client, and the container still ships its whole journal"; + ok = + let + env = atticAlone.systemd.services.swarm-journal-forwarder-oidc.environment; + inner = atticAlone.containers.swarm-attic.config.services.opentelemetry-collector.settings; + in + env.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/attic.pem" + && env.BAO_CLIENT_KEY == "/var/lib/swarm-bao-pki/attic-key.pem" + && + (forwarderOf atticAlone).store.secretPath + == "secret/swarm/services/swarm-attic-collector/oidc/client" + && !(inner.receivers.journald ? units); + } { # The hive collector ships the container units' start output (the rootfs # check's lines) unless the swarm collector already reads the whole