forward the attic container's journal to the swarm collector

nix/host-modules/default.nix

@@ -52,6 +52,7 @@
     ./glue-ci-runner-otel-oidc-client.nix
     ./glue-claude-login-oidc-client.nix
     ./glue-notify-router-oidc-client.nix
+    ./glue-attic-otel-oidc-client.nix
     ./glue-forge-otel-oidc-client.nix
     ./glue-grafana-otel-oidc-client.nix
     ./glue-matrix-otel-oidc-client.nix

nix/host-modules/glue-attic-otel-oidc-client.nix

@@ -0,0 +1,37 @@
+# Glue: register the cache container's journal forwarder as an OIDC client
+# wherever authelia runs.
+#
+# ONE PAIRING PER FILE: that forwarder ← authelia, and nothing else.
+# Without it authelia mints no secret, the publisher has nothing to copy into
+# the store, and the forwarder's export is a 401.
+#
+# Gated on authelia being HERE, not on this host running the cache: a client is
+# a row in THIS host's provider config, the split
+# ./glue-victorialogs-otel-oidc-client.nix makes for the same reason.
+{
+  lib,
+  config,
+  ...
+}:
+let
+  hyperhiveCfg = config.services.hyperhive;
+  clientId = hyperhiveCfg.swarm.attic.otel.clientId;
+in
+{
+  config = lib.mkIf hyperhiveCfg.deploy.authelia.enable {
+    services.hyperhive.swarm.authelia.oidc.clients = [
+      {
+        id = clientId;
+        description = "HyperHive cache journal forwarder";
+        kind = "machine";
+        redirectUris = [ ];
+        # Its own id as its own permitted audience, the one the swarm
+        # collector's receiver for this container checks.
+        audience = [ clientId ];
+        # The receiver verifies the token offline against `/jwks.json`;
+        # authelia's default opaque handle fails there.
+        accessTokenSignedResponseAlg = "RS256";
+      }
+    ];
+  };
+}

nix/host-modules/swarm-attic-options.nix

@@ -13,6 +13,21 @@ let
 in
 {
   options.services.hyperhive.swarm.attic = {
+    otel.clientId = lib.mkOption {
+      type = lib.types.str;
+      readOnly = true;
+      default = "swarm-attic-collector";
+      description = ''
+        OAuth2 client id the journal forwarder inside the cache's container
+        authenticates as, and the audience it asks its token for.
+
+        Read-only because three hosts have to agree on it: authelia registers
+        the client (`glue-attic-otel-oidc-client.nix`), the swarm collector
+        admits it on the container's receiver (`swarm.otel.serviceProducers`),
+        and the secret publisher carries its secret to the store.
+      '';
+    };
+
     ownDomain = lib.mkOption {
       type = lib.types.str;
       readOnly = true;

nix/host-modules/swarm-attic.nix

@@ -29,6 +29,9 @@ let
   atticCfg = hyperhiveCfg.swarm.attic;
   baoCfg = hyperhiveCfg.swarm.bao;
   baoDeploy = hyperhiveCfg.deploy.bao;
+  autheliaCfg = hyperhiveCfg.swarm.authelia;
+  networkCfg = hyperhiveCfg.network;
+  gatewayCfg = hyperhiveCfg.gateway;
 
   machine = "swarm-attic";
   unit = "swarm-attic";
@@ -87,6 +90,33 @@ let
 
   haveClientIdentity = baoDeploy.atticClientCertFile != null && baoDeploy.atticClientKeyFile != null;
 
+  # The container's journal forwarder pushes to its own receiver on the swarm
+  # collector, the route `serviceProducers.swarm-attic` in
+  # ./swarm-otel-service.nix declares.
+  forwarderClientId = atticCfg.otel.clientId;
+  forwarderEndpoint = "https://${hyperhiveCfg.swarm.otel.domain}/${machine}";
+  forwarderUnit = "swarm-journal-forwarder-oidc";
+  # In the host netns: distinct from every other collector's there.
+  forwarderTelemetryPort = 8903;
+  forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
+  # The cache's store identity, handed into the container as system credentials
+  # for the forwarder's fetch unit. The host units keep reading the same files.
+  forwarderStoreFiles = lib.filterAttrs (_: source: source != null) {
+    attic-bao-cert = baoDeploy.atticClientCertFile;
+    attic-bao-key = baoDeploy.atticClientKeyFile;
+    attic-bao-server-ca = baoDeploy.serverCaFile;
+  };
+  # nixos-container expands `extraFlags` unquoted, so a source path must be
+  # one word.
+  oneWord = path: builtins.match ".*[[:space:]].*" path == null;
+
+  # The forwarder reaches the swarm collector and authelia by name over TLS; in
+  # self-signed mode their leaves chain to the hive CA.
+  caTrust = import ./lib/hive-ca-trust.nix {
+    inherit lib gatewayCfg;
+    tlsCfg = hyperhiveCfg.deploy.hive-controller.tls;
+  };
+
   # Both host units log in as the cache's principal.
   storeEnvironment = {
     BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
@@ -171,6 +201,35 @@ in
           is set to null.
         '';
       }
+      {
+        assertion = lib.all oneWord (lib.attrValues forwarderStoreFiles);
+        message = ''
+          The cache's store identity paths must not contain whitespace:
+          nspawn's --load-credential receives them through nixos-container's
+          unquoted extra flags.
+        '';
+      }
+      {
+        assertion =
+          (hyperhiveCfg.swarm.otel.serviceProducers.${machine}.clientId or null) == forwarderClientId;
+        message = ''
+          services.hyperhive.swarm.otel.serviceProducers has no entry
+          "${machine}" with clientId "${forwarderClientId}", so the swarm
+          collector has no receiver for the cache's journal forwarder.
+          Declare it in swarm-otel-service.nix.
+        '';
+      }
+    ];
+
+    # nspawn fails to start the container when a credential source is
+    # missing, so where this host mints the leaf the container waits for it. The
+    # CA ordering keeps the trust bundle's bind source there first.
+    systemd.services."container@${machine}" = lib.mkMerge [
+      caTrust.containerOrdering
+      {
+        after = [ "swarm-bao-pki.service" ];
+        wants = [ "swarm-bao-pki.service" ];
+      }
     ];
 
     services.hyperhive.gateway.localNames = [ atticCfg.domain ];
@@ -279,19 +338,71 @@ in
     containers.${machine} = {
       autoStart = true;
       ephemeral = false;
+      # No `--link-journal=host`: the journal stays in the guest, where the
+      # container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
+      extraFlags = lib.mapAttrsToList (
+        id: source: "--load-credential=${id}:${source}"
+      ) forwarderStoreFiles;
       privateNetwork = false;
-      bindMounts.${secretDir} = {
-        hostPath = secretDir;
-        isReadOnly = true;
-      };
+      bindMounts = {
+        ${secretDir} = {
+          hostPath = secretDir;
+          isReadOnly = true;
+        };
+      }
+      // caTrust.bindMount;
 
       config =
         { ... }:
         {
-          imports = [ ../container-modules/swarm-container.nix ];
+          imports = [
+            ../container-modules/swarm-container.nix
+            ../container-modules/swarm-journal-forwarder.nix
+            (import ../container-modules/swarm-container-resolver.nix {
+              inherit (networkCfg) bridgeIp;
+              # The forwarder and its secret fetch dial the swarm collector,
+              # authelia and the store by name.
+              dnsConsumers = [
+                "opentelemetry-collector.service"
+                "${forwarderUnit}.service"
+              ];
+            })
+            # Go: `SSL_CERT_FILE` replaces its trust store, for the swarm
+            # collector's and authelia's names.
+            (caTrust.trustBundle {
+              inherit pkgs;
+              name = machine;
+              consumers = [ "opentelemetry-collector" ];
+            })
+          ];
 
           services.hyperhive.swarmContainer.privateNetwork = false;
 
+          # Authenticated as the forwarder's own OIDC client, with the cache's
+          # store identity: one identity for the whole container. It is a
+          # system credential, so the fetch unit loads it.
+          services.hyperhive.swarmJournalForwarder = {
+            enable = true;
+            serviceName = "attic";
+            endpoint = forwarderEndpoint;
+            clientId = forwarderClientId;
+            tokenUrl = "${toString autheliaCfg.url}/api/oidc/token";
+            telemetryPort = forwarderTelemetryPort;
+            store = {
+              package = baoDeploy.package;
+              addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
+              secretPath = "secret/swarm/services/${forwarderClientId}/oidc/client";
+              clientCertFile = forwarderCredential "attic-bao-cert";
+              clientKeyFile = forwarderCredential "attic-bao-key";
+              caFile =
+                if forwarderStoreFiles ? attic-bao-server-ca then
+                  forwarderCredential "attic-bao-server-ca"
+                else
+                  null;
+            };
+          };
+          systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames forwarderStoreFiles;
+
           systemd.services.${unit} = {
             description = "attic binary cache server";
             wantedBy = [ "multi-user.target" ];

nix/host-modules/swarm-bao.nix

@@ -1187,7 +1187,9 @@ let
 
   # The in-swarm cache's three secrets, create and read and nothing more: its
   # host units create each one only while it is absent, so none needs `update`,
-  # and a rewrite of a signing key or a token an operator holds is refused.
+  # and a rewrite of a signing key or a token an operator holds is refused. The
+  # cache container's journal forwarder reads its client secret under the same
+  # leaf.
   atticWriters = [
     {
       name = "swarm-attic";
@@ -1203,7 +1205,8 @@ let
             atticJwtLeaf
             atticPushTokenLeaf
             atticPullTokenLeaf
-          ];
+          ]
+        + readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.attic.otel.clientId}/oidc/client";
     }
   ];
 

nix/host-modules/swarm-otel-service.nix

@@ -304,8 +304,12 @@ in
     swarm-notify-router = {
       clientId = hyperhiveCfg.swarm.notify-router.clientId;
     };
-    # The forwarders in ./swarm-ci-runner.nix, ./swarm-grafana.nix and
-    # ./swarm-victorialogs.nix, each authenticated as a client of its own.
+    # The forwarders in ./swarm-attic.nix, ./swarm-ci-runner.nix,
+    # ./swarm-grafana.nix and ./swarm-victorialogs.nix, each authenticated as a
+    # client of its own.
+    swarm-attic = {
+      clientId = hyperhiveCfg.swarm.attic.otel.clientId;
+    };
     swarm-ci-runner = {
       clientId = hyperhiveCfg.swarm.ci-runner.otel.clientId;
     };

nix/host-modules/swarm-otel.nix

@@ -202,7 +202,7 @@ let
   # `swarm-builder`, `swarm-claude-login`, `swarm-notify-router`,
   # `swarm-review-classifier`, `swarm-ci-runner`, `swarm-grafana`,
   # `swarm-victorialogs`, `hive-forge`, `hive-matrix`, `swarm-victoriametrics`,
-  # `swarm-authelia`) forwards its own
+  # `swarm-authelia`, `swarm-attic`) forwards its own
   # journal from its own collector, so this receiver never sees their units.
   hostJournalDir = "/var/log/journal";
   hostJournalMount = "/run/host-journal";

nix/host-modules/swarm-secret-publisher.nix

@@ -55,8 +55,9 @@ let
     # The notification router, which reads it itself inside its container.
     hyperhiveCfg.swarm.notify-router.clientId
     # The journal forwarders in the CI runner's, Grafana's, the log store's,
-    # the forge's and the homeserver's containers, each reading it inside its
-    # container.
+    # the cache's, the forge's and the homeserver's containers, each reading it
+    # inside its container.
+    hyperhiveCfg.swarm.attic.otel.clientId
     hyperhiveCfg.swarm.ci-runner.otel.clientId
     hyperhiveCfg.swarm.grafana.otel.clientId
     hyperhiveCfg.swarm.victorialogs.otel.clientId