matrix: report homeserver up, latency and database size through existing collectors

tuwunel 1.9.4 exports no metrics. The host collector gets an httpcheck
receiver on the gateway's /_matrix/client/versions (new
services.hyperhive.otel.httpChecks, declared by hive-matrix), and a timer in
the hive-matrix container pushes the database directory's size through the
forwarder's OTLP port, as the attic cache-size unit does. The size step
dereferences the state directory's symlink and pushes nothing for a result
under 4096 bytes.

docs/swarm/services.md

@@ -399,6 +399,25 @@ authelia, cache, forge and homeserver have a forwarder client of their own
 `tuwunel`) and no `hive` label. The forwarder of all but the store's is
 `nix/container-modules/swarm-journal-forwarder.nix`.
 
+**The homeserver's metrics.** tuwunel 1.9.4 exports no metrics of its own, so
+request rates, sync load and federation traffic aren't available. Two series
+stand in for them, and neither adds a service:
+
+- The hive collector on the homeserver's host requests
+  `https://<gatewayHost>/_matrix/client/versions` every 60 s with its
+  `httpcheck` receiver (`services.hyperhive.otel.httpChecks`). It reports
+  `httpcheck.status` (one series per status class), `httpcheck.duration` and
+  `httpcheck.error`, labelled with the URL. The container's forwarder can't
+  make this request, because `httpcheck` dials HTTP and HTTPS only and tuwunel
+  listens on a unix socket. The request passes through the gateway, so it
+  covers the gateway's route as well as the homeserver.
+- The `hive-matrix-database-size` timer in the container runs every five
+  minutes. It measures tuwunel's database directory and pushes the gauge
+  `hyperhive.matrix.database_bytes` through the forwarder's OTLP port,
+  stamped `service.name` `hive-matrix`. It exists only where the container
+  has a forwarder, which needs matrix-ctl's store identity set. A size under
+  4096 bytes is no database, so the timer logs a warning and pushes nothing.
+
 The swarm collector's own container, `swarm-otel`, has no forwarder: its
 collector reads the container's journal itself (receiver `journald/self`) and
 exports it to the log store like the host's journal, stamped `service.name`

nix/checks.nix

@@ -77,6 +77,9 @@ in
   # Runs ./host-modules/lib/refresh-consumer.nix's restart decision under a
   # comma-decimal locale, against a stub `systemctl`.
   script-test-refresh-consumer = import ./script-tests/refresh-consumer.nix { inherit pkgs; };
+  # Runs ./host-modules/lib/database-size-push.nix against a symlinked directory,
+  # as a dynamic user's state directory is: the reported size is the target's.
+  script-test-database-size-push = import ./script-tests/database-size-push.nix { inherit pkgs; };
   # Runs ./host-modules/lib/retry-in-job.nix's `retry_in_job`, which the agent
   # trust-bundle fetch retries through inside its start job: success after two
   # failures, the wait schedule, and a return of 1 once the budget is spent.

nix/host-modules/hive-matrix.nix

@@ -153,6 +153,14 @@ let
   # Loopback in the container's own netns, and distinct from the other service
   # containers' forwarders all the same.
   forwarderTelemetryPort = 8899;
+  # Loopback in the container's own netns: the forwarder's OTLP/HTTP metrics
+  # receiver, which the database-size exporter pushes to.
+  forwarderOtlpPort = 8906;
+  databaseSizeUnit = "hive-matrix-database-size";
+  databaseSizePush = import ./lib/database-size-push.nix { inherit lib; };
+  # nixpkgs' default `services.matrix-tuwunel.stateDirectory`, which is also
+  # the database directory under /var/lib.
+  stateName = "tuwunel";
 
   # ── the swarm's own appservice ──────────────────────────────────────
   #
@@ -497,6 +505,13 @@ in
       }
     '';
 
+    # The collector requests the homeserver through the gateway: tuwunel
+    # exports no metrics and binds no TCP port, and `httpcheck` dials HTTP(S)
+    # only, so the container's own forwarder cannot reach the socket.
+    services.hyperhive.otel.httpChecks = lib.optional (
+      cfg.gatewayHost != null
+    ) "https://${cfg.gatewayHost}/_matrix/client/versions";
+
     # `server_name = gatewayHost`. `/_matrix/*` → tuwunel (CORS `*`, 50M
     # body cap, 1h long-poll timeout). `/` serves fluffychat. nginx's
     # longest-prefix rule puts `/_matrix/`
@@ -883,6 +898,7 @@ in
             clientId = forwarderClientId;
             tokenUrl = "${toString autheliaUrl}/api/oidc/token";
             telemetryPort = forwarderTelemetryPort;
+            otlpMetricsPort = forwarderOtlpPort;
             store = {
               package = deployCfg.bao.package;
               addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
@@ -893,6 +909,61 @@ in
             };
           };
 
+          # The size of tuwunel's database directory, pushed to the forwarder.
+          # tuwunel exports no metrics of its own. It runs as tuwunel's dynamic
+          # user, shared by name, because the state directory is mode 0700;
+          # with tuwunel stopped the name gets another uid and the read fails.
+          systemd.services.${databaseSizeUnit} = lib.mkIf ctlActive {
+            description = "push the size of tuwunel's database directory to the swarm collector";
+            wants = [ "tuwunel.service" ];
+            after = [
+              "tuwunel.service"
+              "opentelemetry-collector.service"
+            ];
+            path = [
+              pkgs.coreutils
+              pkgs.curl
+            ];
+            serviceConfig = {
+              Type = "oneshot";
+              User = "tuwunel";
+              DynamicUser = true;
+              StateDirectory = stateName;
+              StateDirectoryMode = "0700";
+              SyslogIdentifier = databaseSizeUnit;
+              UMask = "0077";
+              CapabilityBoundingSet = [ "" ];
+              NoNewPrivileges = true;
+              PrivateDevices = true;
+              PrivateTmp = true;
+              ProtectSystem = "strict";
+              ProtectHome = true;
+              ProtectKernelTunables = true;
+              ProtectKernelModules = true;
+              ProtectControlGroups = true;
+              RestrictAddressFamilies = [
+                "AF_INET"
+                "AF_INET6"
+              ];
+              RestrictNamespaces = true;
+              RestrictRealtime = true;
+              LockPersonality = true;
+              SystemCallArchitectures = "native";
+            };
+            script = databaseSizePush {
+              dir = "/var/lib/${stateName}";
+              otlpPort = forwarderOtlpPort;
+            };
+          };
+
+          systemd.timers.${databaseSizeUnit} = lib.mkIf ctlActive {
+            wantedBy = [ "timers.target" ];
+            timerConfig = {
+              OnBootSec = "2min";
+              OnUnitActiveSec = "5min";
+            };
+          };
+
           # Its own netns, so it keeps its own firewall; resolvconf stays on
           # and serves `networking.nameservers` below.
           services.hyperhive.swarmContainer = {

nix/host-modules/lib/database-size-push.nix

@@ -0,0 +1,37 @@
+# The hive-matrix container's database-size step: measures a directory and
+# pushes the byte count as the gauge `hyperhive.matrix.database_bytes` to the
+# forwarder's OTLP/HTTP metrics receiver on loopback.
+#
+# `dir` is a `DynamicUser` unit's `/var/lib/<name>`, which systemd makes a
+# symlink to `/var/lib/private/<name>`. `--dereference-args` measures the
+# directory behind the link and works unchanged when `dir` is a real
+# directory; without it `du` reports the link itself (a few bytes).
+#
+# A result under 4096 bytes is not a database, since a directory entry alone
+# is that big: the step warns on stderr and pushes nothing, so the gauge
+# stays absent rather than wrong.
+#
+# Pure function — NOT a NixOS module. Needs `du`, `cut`, `date` and `curl`.
+{ lib }:
+{ dir, otlpPort }:
+''
+  set -euo pipefail
+
+  bytes="$(du --summarize --bytes --dereference-args ${lib.escapeShellArg dir} | cut -f1)"
+  case "$bytes" in
+    "" | *[!0-9]*)
+      echo "du answered a non-number: bytes='$bytes'" >&2
+      exit 1
+      ;;
+  esac
+  if [ "$bytes" -lt 4096 ]; then
+    echo "warning: ${dir} measures $bytes bytes, too small for a database; pushing nothing" >&2
+    exit 0
+  fi
+
+  printf '{"resourceMetrics":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"hive-matrix"}}]},"scopeMetrics":[{"metrics":[{"name":"hyperhive.matrix.database_bytes","gauge":{"dataPoints":[{"asInt":"%s","timeUnixNano":"%s"}]}}]}]}]}' \
+    "$bytes" "$(date +%s%N)" |
+    curl --fail --silent --show-error --max-time 20 \
+      -H 'Content-Type: application/json' --data-binary @- \
+      http://127.0.0.1:${toString otlpPort}/v1/metrics
+''

nix/host-modules/otel.nix

@@ -189,6 +189,23 @@ in
       '';
     };
 
+    httpChecks = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      example = [ "https://chat.example/_matrix/client/versions" ];
+      description = ''
+        URLs this hive's collector requests every 60 s with the `httpcheck`
+        receiver, which reports whether each answered (`httpcheck.status`,
+        by status class), how long it took (`httpcheck.duration`) and
+        whether the request failed (`httpcheck.error`).
+
+        **A service adds its own URL, from its own module, under its own
+        `enable`.** The collector dials from the host, so the name has to
+        resolve there and its certificate has to chain to the trust bundle
+        this collector reads. Empty renders no receiver.
+      '';
+    };
+
     journalUnits = lib.mkOption {
       type = lib.types.listOf lib.types.str;
       default = [ ];
@@ -514,6 +531,15 @@ in
           # validates and starts perfectly while reading nothing — and the
           # empty set is the default, so that inert shape would be what
           # most hives deploy.
+          // lib.optionalAttrs (otel.httpChecks != [ ]) {
+            httpcheck = {
+              collection_interval = "60s";
+              targets = map (endpoint: {
+                inherit endpoint;
+                method = "GET";
+              }) otel.httpChecks;
+            };
+          }
           // lib.optionalAttrs (otel.scrapeTargets != { }) {
             prometheus.config.scrape_configs = lib.mapAttrsToList (job: target: {
               job_name = job;
@@ -638,6 +664,7 @@ in
               "hostmetrics"
             ]
             ++ lib.optional (otel.scrapeTargets != { }) "prometheus"
+            ++ lib.optional (otel.httpChecks != [ ]) "httpcheck"
             ++ lib.optional gatewayCfg.enable "nginx";
             # Applies to EVERY receiver above, not just `hostmetrics` — a
             # processor sits in the pipeline, not on a receiver. That is

nix/module-eval/matrix-core.nix

@@ -178,6 +178,65 @@ let
         && !(apex ? "= /.well-known/matrix/client")
         && !(apex ? "= /.well-known/matrix/server");
     }
+    {
+      # tuwunel has no metrics of its own, so the hive collector requests
+      # `/_matrix/client/versions` through the gateway. The URL carries no
+      # port, like the rest of this module's URLs. The elsewhere host is the
+      # control: the same lookups there find no check, and the receiver stays
+      # out of its pipeline.
+      name = "the hive collector requests the homeserver's versions endpoint through the gateway, only where the homeserver runs";
+      ok =
+        let
+          url = "https://chat.t.local/_matrix/client/versions";
+          settings = m: m.services.opentelemetry-collector.settings;
+          pipeline = m: (settings m).service.pipelines.metrics.receivers;
+        in
+        matrixNoBaoIdentity.services.hyperhive.otel.httpChecks == [ url ]
+        &&
+          (settings matrixNoBaoIdentity).receivers.httpcheck.targets == [
+            {
+              endpoint = url;
+              method = "GET";
+            }
+          ]
+        && lib.elem "httpcheck" (pipeline matrixNoBaoIdentity)
+        && matrixElsewhere.services.hyperhive.otel.httpChecks == [ ]
+        && !((settings matrixElsewhere).receivers ? httpcheck)
+        && !(lib.elem "httpcheck" (pipeline matrixElsewhere))
+        # Presence control for the two lookups above.
+        && (settings matrixElsewhere).receivers ? hostmetrics
+        && lib.elem "hostmetrics" (pipeline matrixElsewhere);
+    }
+    {
+      # The size is read from the database directory, as the dynamic user the
+      # homeserver runs as, and pushed to the forwarder's OTLP port. Without a
+      # store identity the container has no forwarder, so there is nothing to
+      # push to and neither unit appears; the hive with the identity is the
+      # control that the same lookups find them.
+      name = "the homeserver container pushes its database size through the forwarder, and only with a forwarder";
+      ok =
+        let
+          inner = m: m.containers.hive-matrix.config;
+          here = inner matrixBodyCap;
+          without = inner matrixNoBaoIdentity;
+          svc = here.systemd.services.hive-matrix-database-size;
+          f = here.services.hyperhive.swarmJournalForwarder;
+        in
+        svc.serviceConfig.Type == "oneshot"
+        && svc.serviceConfig.DynamicUser
+        && svc.serviceConfig.User == "tuwunel"
+        && svc.serviceConfig.StateDirectory == here.services.matrix-tuwunel.stateDirectory
+        && lib.hasInfix "du --summarize --bytes --dereference-args /var/lib/tuwunel" svc.script
+        && lib.hasInfix "-lt 4096" svc.script
+        && lib.hasInfix "hyperhive.matrix.database_bytes" svc.script
+        && lib.hasInfix "127.0.0.1:${toString f.otlpMetricsPort}/v1/metrics" svc.script
+        && f.otlpMetricsPort != null
+        && here.systemd.timers.hive-matrix-database-size.timerConfig.OnUnitActiveSec == "5min"
+        && !(without.systemd.services ? hive-matrix-database-size)
+        && !(without.systemd.timers ? hive-matrix-database-size)
+        # Presence control: the container's units are found by the same lookup.
+        && without.systemd.services ? tuwunel;
+    }
   ];
 in
 runGroup "matrix-core" cases

nix/script-tests/database-size-push.nix

@@ -0,0 +1,21 @@
+# `checks.script-test-database-size-push` — runs ../host-modules/lib/database-size-push.nix's
+# script against a symlinked directory, as a `DynamicUser` unit's state directory
+# is, with a stub `curl`. The cases are in ./database-size-push.sh.
+{
+  pkgs,
+  ...
+}:
+pkgs.runCommand "hyperhive-script-test-database-size-push"
+  {
+    nativeBuildInputs = [ pkgs.coreutils ];
+    SCRIPT_LINK = pkgs.writeText "database-size-push-link.sh" (
+      import ../host-modules/lib/database-size-push.nix { inherit (pkgs) lib; } {
+        dir = "@DIR@";
+        otlpPort = 8906;
+      }
+    );
+  }
+  ''
+    ${pkgs.bash}/bin/bash ${./database-size-push.sh}
+    touch "$out"
+  ''

nix/script-tests/database-size-push.sh

@@ -0,0 +1,74 @@
+# Cases for ./database-size-push.nix. The directory is reached through a
+# symlink, as a dynamic user's state directory is. `curl` is a stub that
+# records the payload and the URL it is given.
+set -uo pipefail
+
+failures=0
+
+fail() {
+  echo "FAILED: $case: $*" >&2
+  failures=$((failures + 1))
+}
+
+root=$(mktemp -d)
+mkdir -p "$root/bin" "$root/private/tuwunel"
+head -c 100000 /dev/zero >"$root/private/tuwunel/db"
+ln -s "$root/private/tuwunel" "$root/tuwunel"
+
+cat >"$root/bin/curl" <<STUB
+#!$BASH
+printf '%s\n' "\$*" >"$root/curl.args"
+cat >"$root/curl.payload"
+STUB
+chmod +x "$root/bin/curl"
+
+# Runs the script for \$1, returns its status; stdout and stderr land in files.
+run() {
+  rm -f "$root/curl.args" "$root/curl.payload"
+  sed "s|@DIR@|$1|" "$SCRIPT_LINK" >"$root/script.sh"
+  PATH="$root/bin:$PATH" bash "$root/script.sh" >"$root/out" 2>"$root/err"
+}
+
+case="a symlinked directory reports the target's size"
+want=$(du --summarize --bytes "$root/private/tuwunel/" | cut -f1)
+link=$(du --summarize --bytes "$root/tuwunel" | cut -f1)
+[ "$want" -ge 100000 ] || fail "fixture target is $want bytes"
+[ "$link" -lt 4096 ] || fail "fixture link measures $link bytes, so it does not tell the cases apart"
+if run "$root/tuwunel"; then
+  grep -q "\"asInt\":\"$want\"" "$root/curl.payload" || fail "payload is $(cat "$root/curl.payload"), want asInt $want"
+  grep -q 'hyperhive.matrix.database_bytes' "$root/curl.payload" || fail "no metric name in the payload"
+  grep -q '127.0.0.1:8906/v1/metrics' "$root/curl.args" || fail "wrong url: $(cat "$root/curl.args")"
+else
+  fail "the script failed: $(cat "$root/err")"
+fi
+
+case="a real directory reports its own size"
+if run "$root/private/tuwunel"; then
+  grep -q "\"asInt\":\"$want\"" "$root/curl.payload" || fail "payload is $(cat "$root/curl.payload"), want asInt $want"
+else
+  fail "the script failed: $(cat "$root/err")"
+fi
+
+# `du` stub answering a link-sized number, as an old command line would.
+case="an implausibly small size warns and pushes nothing"
+cat >"$root/bin/du" <<STUB
+#!$BASH
+printf '15\t%s\n' "\${@: -1}"
+STUB
+chmod +x "$root/bin/du"
+if run "$root/tuwunel"; then
+  [ ! -e "$root/curl.payload" ] || fail "pushed $(cat "$root/curl.payload")"
+  grep -q 'warning' "$root/err" || fail "no warning: $(cat "$root/err")"
+else
+  fail "the script failed: $(cat "$root/err")"
+fi
+rm "$root/bin/du"
+
+case="a missing directory fails and pushes nothing"
+if run "$root/absent"; then
+  fail "the script succeeded"
+fi
+[ ! -e "$root/curl.payload" ] || fail "pushed on a missing directory"
+
+[ "$failures" -eq 0 ] || exit 1
+echo "all cases passed"