swarm-bao: ship the store's logs and metrics

The store declared no journald units and served no metrics: nothing in
`swarm-bao.nix` mentioned either, while every sibling swarm service
declares both.

Metrics get their own loopback listener rather than a flag on the API
one, and that follows from what a scraper can express rather than from
taste: `swarm.otel.scrapeTargets` is `host:port`, plaintext and with no
credential, while the API listener is TLS and demands a client
certificate once a client CA is set. `metrics_only` narrows the new
listener to the metrics path; `prometheus_retention_time` is what serves
the endpoint at all.

Measured against openbao 2.6.2 before writing any of it: the metrics
path answers 200 on such a listener *while the node is sealed and
uninitialised*, 503 on the API listener, and 404 for a non-metrics path
on the metrics listener.

The listener exists only where a collector does — it is unauthenticated
by design for now, and an endpoint with no reader would be exposure
bought for nothing.

The port cannot be the API port + 1: openbao derives every listener's
cluster address as its own port plus one, so that number is already
taken. An assertion says so, since the failure is otherwise a race with
no log line.

Journald units are declared by the module that defines each unit, not
gathered here, matching the option's own rule — a name nothing defines
is silently ignored, so a central list would read as coverage on hives
that have neither glue module.

Refs #3849

nix/host-modules/glue-bao-tls.nix

@@ -87,6 +87,11 @@ in
     # every client certificate already trusting it, so a rebuild that
     # "refreshed" it would lock every reader in the swarm out at once — the
     # same rule the store's TPM PIN unit follows, for a sharper reason.
+    # Declared beside the unit it names, not in the store's module: an entry
+    # exists only where the unit does, and this one is minted by glue that not
+    # every hive runs.
+    services.hyperhive.swarm.otel.journaldUnits = [ "swarm-bao-pki" ];
+
     systemd.services.swarm-bao-pki = {
       description = "mint the swarm secret store's own CA and leaves";
       before = [ "swarm-bao-certs.service" ];

nix/host-modules/glue-matrix-bao-token.nix

@@ -54,6 +54,11 @@ let
 in
 {
   config = lib.mkIf (hyperhiveCfg.enable && haveClientIdentity && deployCfg.matrix.enable) {
+    # Same rule as the unit's own gate: this reader exists on a host that has a
+    # client identity and a homeserver, which is not every host that runs the
+    # store, so the store's module cannot name it.
+    services.hyperhive.swarm.otel.journaldUnits = [ "swarm-bao-matrix-token" ];
+
     systemd.services.swarm-bao-matrix-token = {
       description = "fetch the matrix registration token from the swarm secret store";
       # Every one of these names a unit that exists only where the store runs.

nix/host-modules/swarm-bao.nix

@@ -169,7 +169,45 @@ let
     }
     // listenerTls;
   }
-  // extraListeners;
+  // extraListeners
+  // metricsListener;
+
+  # Metrics get their own listener rather than a flag on the one above, and
+  # that follows from what a scraper can express: `swarm.otel.scrapeTargets`
+  # is `host:port` with no scheme and no credential, while the API listener is
+  # TLS and — once a client CA is set — demands a client certificate. The
+  # collector cannot reach it at all.
+  #
+  # `metrics_only` narrows this one to the metrics path (every other path 404s)
+  # and the unauthenticated access is confined to loopback. **Deliberately
+  # unauthenticated for now** — a tracked follow-up owns giving the collector a
+  # credential, since no scrape option can carry one today.
+  #
+  # Exists only where a collector does: an endpoint with no reader is exposure
+  # bought for nothing.
+  metricsListener = lib.optionalAttrs scrapeHere {
+    metrics = {
+      type = "tcp";
+      address = "127.0.0.1:${toString baoDeploy.metricsPort}";
+      tls_disable = true;
+      telemetry = {
+        unauthenticated_metrics_access = true;
+        metrics_only = true;
+      };
+    };
+  };
+
+  scrapeHere = deployCfg.swarm-otel.enable;
+
+  # Non-zero is what SERVES the endpoint at all — the switch is a duration, not
+  # a boolean, so a zero here is an openbao that answers 404 on a listener
+  # configured to do nothing else.
+  telemetry = lib.optionalAttrs scrapeHere {
+    telemetry = {
+      prometheus_retention_time = "24h";
+      disable_hostname = true;
+    };
+  };
 
   # Raft REFUSES TO START without `cluster_addr`, and the message names neither
   # the setting nor the stanza: "cluster address must be set when using raft
@@ -332,6 +370,25 @@ in
       '';
     };
 
+    metricsPort = lib.mkOption {
+      type = lib.types.port;
+      default = 8202;
+      description = ''
+        Loopback port the store serves its Prometheus metrics on, scraped by
+        a collector on this same host.
+
+        Here and not beside {option}`services.hyperhive.swarm.bao.port`
+        because no *client* of the store ever needs it: reaching this port
+        means being the local collector, which is a property of the host
+        running the store.
+
+        ⚠️ Cannot be {option}`services.hyperhive.swarm.bao.port` **+ 1** —
+        openbao derives every listener's cluster address as its own port plus
+        one, so the API listener already holds that number. The default sits
+        one above it, and its own derived cluster address one above that.
+      '';
+    };
+
     extraListenAddresses = lib.mkOption {
       type = lib.types.listOf lib.types.str;
       default = [ ];
@@ -417,6 +474,16 @@ in
             See https://openbao.org/community/deprecation/
           '';
         }
+        {
+          assertion = baoDeploy.metricsPort != cfg.port + 1;
+          message = ''
+            services.hyperhive.deploy.bao.metricsPort is ${toString baoDeploy.metricsPort},
+            which openbao already uses as the API listener's cluster address
+            (services.hyperhive.swarm.bao.port + 1). Two listeners would claim
+            the same port, and which one wins is a race with nothing in any log
+            about it. Pick any other free port.
+          '';
+        }
         {
           assertion = haveServerTls;
           message = ''
@@ -448,6 +515,23 @@ in
       services.hyperhive.gateway.localNames = [ cfg.domain ];
     })
 
+    (lib.mkIf (hyperhiveCfg.enable && deployCfg.bao.enable) {
+      # The in-container unit plus the two host-side ones this module defines.
+      # `swarm-bao-pki` and `swarm-bao-matrix-token` are declared by the glue
+      # modules that create them, per the option's own rule — and a name
+      # nothing defines is silently ignored, so naming them from here would
+      # read as coverage on hives that have neither.
+      services.hyperhive.swarm.otel.journaldUnits = [
+        "openbao"
+        "swarm-bao-certs"
+        "swarm-bao-token"
+      ];
+
+      services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf scrapeHere {
+        bao = "127.0.0.1:${toString baoDeploy.metricsPort}";
+      };
+    })
+
     (lib.mkIf (hyperhiveCfg.enable && deployCfg.bao.enable && haveServerTls) {
       # Provisions the TPM-backed token the seal above names. One-shot and
       # idempotent on ABSENCE, never on content: regenerating a PIN would
@@ -632,6 +716,7 @@ in
                 storage.raft.path = stateDir;
               }
               // advertise
+              // telemetry
               // sealSettings;
             };
 

nix/module-eval.nix

@@ -109,6 +109,17 @@ let
     deploy.bao.enable = true;
     deploy.bao.extraListenAddresses = [ "10.0.0.1" ];
   };
+  # The store with and without a collector on the same host. `scrapeTargets`
+  # is only ever read by a local collector, so the metrics endpoint is a
+  # function of the pairing rather than of the store.
+  baoWithCollector = hive {
+    deploy.bao.enable = true;
+    deploy.swarm-otel.enable = true;
+  };
+  baoNoCollector = hive {
+    deploy.bao.enable = true;
+    deploy.swarm-otel.enable = false;
+  };
 
   # The config file openbao parses, not the nix that produces it: a setting it
   # requires is absent here without anything in the module system minding, so
@@ -286,6 +297,26 @@ let
       name = "a declared extra address renders a second listener beside loopback";
       ok = builtins.length (builtins.attrNames (baoSettings baoTwoAddresses).listener) == 2;
     }
+    {
+      # Retention is what serves the endpoint at all, so the listener alone
+      # would be a port that answers 404.
+      name = "a store beside a collector serves metrics on its own listener";
+      ok =
+        let
+          s = baoSettings baoWithCollector;
+        in
+        s.listener ? metrics && (s.telemetry.prometheus_retention_time or "0s") != "0s";
+    }
+    {
+      # Absence arm. Unauthenticated by design, so it must not exist where
+      # nothing reads it.
+      name = "a store with no collector beside it serves no metrics";
+      ok =
+        let
+          s = baoSettings baoNoCollector;
+        in
+        !(s.listener ? metrics) && !(s ? telemetry);
+    }
   ];
 
   bad = builtins.filter (c: !c.ok) cases;