swarm-victoriametrics: forward the container's journal to the swarm collector

The metrics store's container now runs the journal forwarder the other
service containers run, pushing to a receiver of its own on the swarm
collector (serviceProducers swarm-victoriametrics) as a new machine OIDC
client (swarm.victoriametrics.otel.clientId), registered by a glue file
wherever authelia runs and published to the store by the secret publisher.

The container keeps its loopback-only netns. The collector cannot dial a
unix socket, so swarm-victoriametrics-egress-relay, a socket-activated
systemd-socket-proxyd inside the container, takes its connections on
127.0.0.1:4380 and passes them to /run/swarm-victoriametrics-egress/egress.sock.
The host's nginx listens there and sends /swarm-victoriametrics/ to the swarm
collector and /api/oidc/token to authelia, both by name and verified, and
refuses every other path. nginx gets ReadWritePaths on that directory and
removes a stale socket before it starts.

The container cannot reach the store, so the client secret is fetched on the
host by swarm-victoriametrics-forwarder-oidc under swarm-metrics-auth's leaf,
the shape swarm-bao-forwarder-oidc has, and bound into the container
read-only. The forwarder module gains hostSecretFile for that: set, it renders
no fetch unit. swarm-metrics-auth's policy gains one read stanza on the
forwarder's client path.

The container drops --link-journal=host, and the host collector stops reading
it.

Also lists the forge publisher's and matrix-ctl's leaves in the forwarder
secrets row of docs/swarm/secrets.md, which the hive-forge/hive-matrix change
missed.

nix/host-modules/default.nix

@@ -58,6 +58,7 @@
     ./glue-swarm-bao-otel-oidc-client.nix
     ./glue-swarm-otel-oidc-client.nix
     ./glue-victorialogs-otel-oidc-client.nix
+    ./glue-victoriametrics-otel-oidc-client.nix
     ./swarm-authelia-service.nix
     ./swarm-authelia.nix
     ./swarm-bao-service.nix

nix/host-modules/glue-victoriametrics-otel-oidc-client.nix

@@ -0,0 +1,37 @@
+# Glue: register the metrics store container's journal forwarder as an OIDC
+# client wherever authelia runs.
+#
+# ONE PAIRING PER FILE: that forwarder ← authelia, and nothing else.
+# Without it authelia mints no secret, the publisher has nothing to copy into
+# the store, and the forwarder's export is a 401.
+#
+# Gated on authelia being HERE, not on this host running the metrics store: a
+# client is a row in THIS host's provider config, the split
+# ./glue-swarm-bao-otel-oidc-client.nix makes for the same reason.
+{
+  lib,
+  config,
+  ...
+}:
+let
+  hyperhiveCfg = config.services.hyperhive;
+  clientId = hyperhiveCfg.swarm.victoriametrics.otel.clientId;
+in
+{
+  config = lib.mkIf hyperhiveCfg.deploy.authelia.enable {
+    services.hyperhive.swarm.authelia.oidc.clients = [
+      {
+        id = clientId;
+        description = "HyperHive metrics store journal forwarder";
+        kind = "machine";
+        redirectUris = [ ];
+        # Its own id as its own permitted audience, the one the swarm
+        # collector's receiver for this container checks.
+        audience = [ clientId ];
+        # The receiver verifies the token offline against `/jwks.json`;
+        # authelia's default opaque handle fails there.
+        accessTokenSignedResponseAlg = "RS256";
+      }
+    ];
+  };
+}

nix/host-modules/swarm-bao.nix

@@ -1038,12 +1038,16 @@ let
   ];
 
   # The metrics store's responder, which checks the bearer Grafana presents
-  # against the one stored for it, and reads nothing else.
+  # against the one stored for it; and the host unit that fetches the store
+  # container's journal forwarder secret under the same leaf. Nothing else.
   metricsAuthReaders = [
     {
       name = "swarm-metrics-auth";
       cn = baoDeploy.metricsAuthCommonName;
-      policyText = readStanza "${credentialMountPath}/data/${grafanaMetricsBearerLeaf}";
+      policyText =
+        readStanza "${credentialMountPath}/data/${grafanaMetricsBearerLeaf}"
+        # The metrics store container's journal forwarder, fetched on its host.
+        + readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.victoriametrics.otel.clientId}/oidc/client";
     }
   ];
 
@@ -1944,7 +1948,10 @@ in
         Subject the store's `swarm-metrics-auth` cert-auth role accepts: the
         identity the metrics store's Grafana auth responder presents to read
         the bearer Grafana's datasource sends,
-        `swarm/services/<grafana client id>/metrics/bearer`, and nothing else.
+        `swarm/services/<grafana client id>/metrics/bearer`, and the one
+        `swarm-victoriametrics-forwarder-oidc` presents to read
+        `swarm/services/<metrics store forwarder client id>/oidc/client`.
+        Nothing else.
 
         ⚠️ Reserved as a hive name by ./swarm.nix, like its siblings.
       '';

nix/host-modules/swarm-otel-service.nix

@@ -439,6 +439,12 @@ in
       clientId = hyperhiveCfg.swarm.matrix.otel.clientId;
       port = lib.mkDefault 4400;
     };
+    # The forwarder in ./swarm-victoriametrics.nix, which reaches this route
+    # through its host's nginx.
+    swarm-victoriametrics = {
+      clientId = hyperhiveCfg.swarm.victoriametrics.otel.clientId;
+      port = lib.mkDefault 4401;
+    };
   };
 
   # Not under any `enable`: see `publishedScrapeTargets` for why every host

nix/host-modules/swarm-otel.nix

@@ -216,9 +216,9 @@ let
   # The `serviceProducers` containers (`swarm-bao`, `swarm-nats`,
   # `swarm-builder`, `swarm-claude-login`, `swarm-review-classifier`,
   # `swarm-ci-runner`, `swarm-grafana`, `swarm-victorialogs`, `hive-forge`,
-  # `hive-matrix`) are the exceptions on purpose: their journals stay inside
-  # the container for its own collector instead, so this receiver never sees
-  # their units.
+  # `hive-matrix`, `swarm-victoriametrics`) are the exceptions on purpose: their
+  # journals stay inside the container for its own collector instead, so this
+  # receiver never sees their units.
   hostJournalDir = "/var/log/journal";
 
   # Each store's OTLP route, by domain. One binding because the same string is

nix/host-modules/swarm-secret-publisher.nix

@@ -106,6 +106,8 @@ let
     hyperhiveCfg.swarm.victorialogs.otel.clientId
     hyperhiveCfg.swarm.forge.otel.clientId
     hyperhiveCfg.swarm.matrix.otel.clientId
+    # The metrics store container's journal forwarder, whose host reads it.
+    hyperhiveCfg.swarm.victoriametrics.otel.clientId
   ];
 
   # The swarm controller's OIDC client, which it reads back from the store

nix/host-modules/swarm-victoriametrics-service.nix

@@ -31,6 +31,22 @@ in
       '';
     };
 
+    otel.clientId = lib.mkOption {
+      type = lib.types.str;
+      readOnly = true;
+      default = "swarm-victoriametrics-collector";
+      description = ''
+        OAuth2 client id the journal forwarder inside the metrics store's
+        container authenticates as, and the audience it asks its token for.
+
+        Read-only because three hosts have to agree on it: authelia
+        registers the client (`glue-victoriametrics-otel-oidc-client.nix`),
+        the swarm collector admits it on the container's receiver
+        (`swarm.otel.serviceProducers`), and the secret publisher carries
+        its secret to the store.
+      '';
+    };
+
     domain = lib.mkOption {
       type = lib.types.str;
       default = "metrics.${domainBase}";

nix/host-modules/swarm-victoriametrics.nix

@@ -22,8 +22,9 @@ let
   gatewayCfg = hyperhiveCfg.gateway;
 
   # A netns of its own with no interface but loopback: the store makes no
-  # outbound calls, and the gateway reaches it over `socketPath` below, so
-  # nothing on the host or the hive bridge can dial its listener.
+  # outbound calls, the gateway reaches it over `socketPath` below, and the
+  # journal forwarder leaves only through `egressSocket`, so nothing on the host
+  # or the hive bridge can dial its listener.
   privateNetwork = true;
 
   # Fixed, not an option: the directory's mode is what keeps the socket
@@ -126,6 +127,72 @@ let
   # The path Grafana's host creates the bearer at, under the KV mount. The
   # same leaf as `grafanaMetricsBearerLeaf` in ./swarm-bao.nix.
   grafanaBearerStorePath = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer";
+
+  # ── the container's journal forwarder ───────────────────────────────
+  #
+  # With no interface but loopback, the forwarder reaches nothing by name. A
+  # relay in the container listens on `relayPort` and forwards each connection
+  # to `egressSocket`, a socket this host's nginx listens on and binds into
+  # the container. nginx sends `/<machine>/` to the swarm collector's route
+  # (`serviceProducers.swarm-victoriametrics` in ./swarm-otel-service.nix) and
+  # the token endpoint to authelia, both by name and verified. The collector
+  # library dials no unix socket itself, so the relay is needed.
+  #
+  # The client secret is fetched on the host, under the responder's store
+  # identity, and bound in read-only: the container cannot reach the store.
+  forwarderClientId = cfg.otel.clientId;
+  gatewayPortSuffix = lib.optionalString (
+    gatewayCfg.httpsPort != 443
+  ) ":${toString gatewayCfg.httpsPort}";
+  otelDomain = hyperhiveCfg.swarm.otel.domain;
+  autheliaDomain = hyperhiveCfg.swarm.authelia.domain;
+
+  # Its own directory, not `socketDir`: that one's mode admits nginx to the
+  # store's listener, and this one admits the container to nginx's.
+  egressDir = "/run/swarm-victoriametrics-egress";
+  egressSocket = "${egressDir}/egress.sock";
+  relayUnit = "swarm-victoriametrics-egress-relay";
+  # Loopback in the container's own netns.
+  relayPort = 4380;
+  relayUrl = "http://127.0.0.1:${toString relayPort}";
+  forwarderTelemetryPort = 8900;
+
+  # The forwarder's way out, see `egressSocket`. Every other path is refused,
+  # so the container reaches the two routes its forwarder uses and nothing
+  # else on the network.
+  egressServer = ''
+    server {
+      listen unix:${egressSocket};
+
+      location / {
+        return 403;
+      }
+
+      location ^~ /${cfg.machine}/ {
+        proxy_pass https://${otelDomain}${gatewayPortSuffix};
+        ${gatewayCfg.lib.verifiedProxyTo otelDomain}
+      }
+
+      location = /api/oidc/token {
+        proxy_pass https://${autheliaDomain};
+        ${gatewayCfg.lib.verifiedProxyTo autheliaDomain}
+      }
+    }
+  '';
+
+  forwarderFetchUnit = "swarm-victoriametrics-forwarder-oidc";
+  forwarderStoreSecretPath = "secret/swarm/services/${forwarderClientId}/oidc/client";
+  # On the host's /run (tmpfs), and seen read-only in the container at the
+  # same path. The file is not created ahead of the fetch: `LoadCredential`
+  # refuses a missing source, so the forwarder stays down until the store
+  # answers and then resumes from its saved read cursor.
+  forwarderSecretDir = "/run/${forwarderFetchUnit}";
+  forwarderSecret = "${forwarderSecretDir}/${forwarderClientId}.secret";
+
+  storeRetry = import ./lib/store-retry.nix { };
+  storeRefreshTimer = import ./lib/store-refresh-timer.nix { };
+  atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
+  refreshConsumer = import ./lib/refresh-consumer.nix { };
 in
 {
   # What the store IS from any hive's point of view, the name it answers on,
@@ -228,6 +295,16 @@ in
           both options.
         '';
       }
+      {
+        assertion =
+          (hyperhiveCfg.swarm.otel.serviceProducers.${cfg.machine}.clientId or null) == forwarderClientId;
+        message = ''
+          services.hyperhive.swarm.otel.serviceProducers has no entry
+          "${cfg.machine}" with clientId "${forwarderClientId}", so the swarm
+          collector has no receiver for the metrics store's journal forwarder.
+          Declare it in swarm-otel-service.nix.
+        '';
+      }
     ];
 
     # The gateway name, inside `deployCfg.victoriametrics.enable` — that
@@ -244,8 +321,9 @@ in
     # Which check a read location's `auth_request` gets. `auth_request` takes a
     # fixed URI, so it names `/__metrics_read_auth`, which rewrites to the
     # target this map picks: a bearer goes to Grafana's check, anything else
-    # (an operator's session cookie) to authelia.
-    services.nginx.appendHttpConfig = ''
+    # (an operator's session cookie) to authelia. After the forwarder's egress
+    # listener, `egressServer`.
+    services.nginx.appendHttpConfig = egressServer + ''
       map $http_authorization $swarm_metrics_read_auth_target {
         "~^Bearer " /__metrics_grafana_auth;
         default /__metrics_authelia;
@@ -359,6 +437,12 @@ in
     # Everything else is kept out.
     systemd.tmpfiles.rules = [
       "d ${socketDir} 0770 root ${toString nginxGid} - -"
+      # The relay in the container connects as root; nothing else on the host
+      # but nginx may.
+      "d ${egressDir} 0750 ${config.services.nginx.user} ${config.services.nginx.group} - -"
+      # The forwarder secret's bind source, so it exists when nspawn sets the
+      # mount up. Not the file: see `forwarderSecretDir`.
+      "d ${forwarderSecretDir} 0700 root root -"
     ];
 
     # Grafana's check on the read API. On the host rather than in the store's
@@ -422,12 +506,100 @@ in
       };
     };
 
+    # nginx runs as its own user under `ProtectSystem=strict`, so it can only
+    # create the socket where it may write. It removes the socket on a clean
+    # exit only; a stale one after a crash fails every later `bind`, and with
+    # it the whole gateway, so it goes before each start. `-`: the directory
+    # is tmpfiles', and absent until that has run.
+    systemd.services.nginx = {
+      serviceConfig.ReadWritePaths = [ "-${egressDir}" ];
+      preStart = lib.mkBefore "rm -f ${egressSocket}";
+    };
+
+    # Reads the forwarder's client secret with the responder's store identity,
+    # the shape ./swarm-bao.nix's `swarm-bao-forwarder-oidc` has. The store
+    # and authelia may be a host away and this boot waits on neither, so a run
+    # that fails retries (./lib/store-retry.nix), and the collector restarts
+    # until the file lands.
+    systemd.services.${forwarderFetchUnit} = {
+      description = "fetch the metrics store forwarder's OIDC client secret from the swarm secret store";
+      after = [
+        "swarm-bao-pki.service"
+        "container@${cfg.machine}.service"
+      ];
+      wants = [ "container@${cfg.machine}.service" ];
+      wantedBy = [ "multi-user.target" ];
+      path = [
+        deployCfg.bao.package
+        pkgs.coreutils
+        pkgs.systemd
+      ];
+      inherit (storeRetry) startLimitBurst startLimitIntervalSec;
+      serviceConfig = storeRetry.serviceConfig // {
+        # No `RemainAfterExit`: the timer below starts this again, and
+        # starting an active unit is a no-op.
+        Type = "oneshot";
+        SyslogIdentifier = forwarderFetchUnit;
+        TimeoutStartSec = 30;
+      };
+      environment = {
+        BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
+        BAO_CLIENT_CERT = deployCfg.victoriametrics.authBaoClientCertFile;
+        BAO_CLIENT_KEY = deployCfg.victoriametrics.authBaoClientKeyFile;
+      }
+      // lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
+        BAO_CACERT = deployCfg.bao.serverCaFile;
+      };
+      script = ''
+        set -euo pipefail
+
+        ${atomicWriteSecret}
+        ${refreshConsumer}
+
+        err="$(mktemp)"
+        trap 'rm -f "$err"' EXIT
+
+        # `-token-only` answers on stdout and skips the token helper, which
+        # needs a `sh` this unit's `path` does not carry.
+        if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
+          echo "could not log in to the swarm secret store with the metrics store responder's certificate." >&2
+          cat "$err" >&2
+          exit 1
+        fi
+        export BAO_TOKEN
+
+        if ! secret="$(bao kv get -field=value ${lib.escapeShellArg forwarderStoreSecretPath} 2>"$err")"; then
+          echo "the store did not return ${forwarderStoreSecretPath}: the forwarder has no client secret yet." >&2
+          cat "$err" >&2
+          exit 1
+        fi
+
+        if [ -z "$secret" ]; then
+          echo "the store returned an empty ${forwarderStoreSecretPath}." >&2
+          exit 1
+        fi
+
+        # `printf` inside `atomic_write_secret` is bash's own, so the value
+        # never becomes an argument in /proc.
+        install -d -m 0700 ${lib.escapeShellArg forwarderSecretDir}
+        if secret_differs ${lib.escapeShellArg forwarderSecret} "$secret"; then
+          atomic_write_secret 0400 root:root ${lib.escapeShellArg forwarderSecret} "$secret"
+        fi
+
+        # `LoadCredential` copies the file at start only: a rotated secret
+        # reaches the forwarder by restarting it.
+        refresh_consumer ${lib.escapeShellArg cfg.machine} opentelemetry-collector.service ${lib.escapeShellArg forwarderSecret}
+      '';
+    };
+
+    systemd.timers.${forwarderFetchUnit} =
+      storeRefreshTimer "re-read the metrics store forwarder's OIDC client secret from the swarm secret store";
+
     containers.${cfg.machine} = {
       autoStart = true;
       ephemeral = false;
-      # Journal files on the host, not inside the container: nixpkgs hardcodes
-      # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
-      extraFlags = [ "--link-journal=host" ];
+      # No `--link-journal=host`: the journal stays in the guest, where the
+      # container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
       inherit privateNetwork;
 
       # A bind mount rather than a path both sides happen to know: a
@@ -436,11 +608,54 @@ in
         hostPath = socketDir;
         isReadOnly = false;
       };
+      # Connecting to a socket is a write to it.
+      bindMounts.${egressDir} = {
+        hostPath = egressDir;
+        isReadOnly = false;
+      };
+      bindMounts.${forwarderSecretDir} = {
+        hostPath = forwarderSecretDir;
+        isReadOnly = true;
+      };
 
       config =
         { config, pkgs, ... }:
         {
-          imports = [ ../container-modules/swarm-container.nix ];
+          imports = [
+            ../container-modules/swarm-container.nix
+            ../container-modules/swarm-journal-forwarder.nix
+          ];
+
+          # Authenticated as the forwarder's own OIDC client, its secret
+          # fetched on the host. Both URLs are the relay's.
+          services.hyperhive.swarmJournalForwarder = {
+            enable = true;
+            serviceName = "victoriametrics";
+            endpoint = "${relayUrl}/${cfg.machine}";
+            clientId = forwarderClientId;
+            tokenUrl = "${relayUrl}/api/oidc/token";
+            telemetryPort = forwarderTelemetryPort;
+            hostSecretFile = forwarderSecret;
+          };
+
+          # The relay out of this netns, started by its first connection.
+          # systemd-socket-proxyd as root: the egress directory admits root
+          # and nginx only.
+          systemd.sockets.${relayUnit} = {
+            description = "loopback listener for the journal forwarder's way out of this container";
+            wantedBy = [ "sockets.target" ];
+            listenStreams = [ "127.0.0.1:${toString relayPort}" ];
+          };
+          systemd.services.${relayUnit} = {
+            description = "relay the journal forwarder's connections to the host's egress socket";
+            requires = [ "${relayUnit}.socket" ];
+            after = [ "${relayUnit}.socket" ];
+            serviceConfig = {
+              ExecStart = "${pkgs.systemd}/lib/systemd/systemd-socket-proxyd ${egressSocket}";
+              PrivateTmp = true;
+              NoNewPrivileges = true;
+            };
+          };
 
           # No resolver unit: with no interface but loopback there is no
           # nameserver to reach, and the store resolves no names.