From bef9d401a46c837a793bf3230746c51bc46332ae Mon Sep 17 00:00:00 2001 From: atlas Date: Thu, 5 Nov 2026 01:24:22 +0100 Subject: [PATCH] matrix: report homeserver up, latency and database size through existing collectors tuwunel 1.9.4 exports no metrics. The host collector gets an httpcheck receiver on the gateway's /_matrix/client/versions (new services.hyperhive.otel.httpChecks, declared by hive-matrix), and a timer in the hive-matrix container pushes the database directory's size through the forwarder's OTLP port, as the attic cache-size unit does. The size step dereferences the state directory's symlink and pushes nothing for a result under 4096 bytes. --- docs/swarm/services.md | 19 ++++++ nix/checks.nix | 3 + nix/host-modules/hive-matrix.nix | 71 ++++++++++++++++++++ nix/host-modules/lib/database-size-push.nix | 37 +++++++++++ nix/host-modules/otel.nix | 27 ++++++++ nix/module-eval/matrix-core.nix | 59 ++++++++++++++++ nix/script-tests/database-size-push.nix | 21 ++++++ nix/script-tests/database-size-push.sh | 74 +++++++++++++++++++++ 8 files changed, 311 insertions(+) create mode 100644 nix/host-modules/lib/database-size-push.nix create mode 100644 nix/script-tests/database-size-push.nix create mode 100644 nix/script-tests/database-size-push.sh diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 362a7a90..7bc604ba 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -399,6 +399,25 @@ authelia, cache, forge and homeserver have a forwarder client of their own `tuwunel`) and no `hive` label. The forwarder of all but the store's is `nix/container-modules/swarm-journal-forwarder.nix`. +**The homeserver's metrics.** tuwunel 1.9.4 exports no metrics of its own, so +request rates, sync load and federation traffic aren't available. Two series +stand in for them, and neither adds a service: + +- The hive collector on the homeserver's host requests + `https:///_matrix/client/versions` every 60 s with its + `httpcheck` receiver (`services.hyperhive.otel.httpChecks`). It reports + `httpcheck.status` (one series per status class), `httpcheck.duration` and + `httpcheck.error`, labelled with the URL. The container's forwarder can't + make this request, because `httpcheck` dials HTTP and HTTPS only and tuwunel + listens on a unix socket. The request passes through the gateway, so it + covers the gateway's route as well as the homeserver. +- The `hive-matrix-database-size` timer in the container runs every five + minutes. It measures tuwunel's database directory and pushes the gauge + `hyperhive.matrix.database_bytes` through the forwarder's OTLP port, + stamped `service.name` `hive-matrix`. It exists only where the container + has a forwarder, which needs matrix-ctl's store identity set. A size under + 4096 bytes is no database, so the timer logs a warning and pushes nothing. + The swarm collector's own container, `swarm-otel`, has no forwarder: its collector reads the container's journal itself (receiver `journald/self`) and exports it to the log store like the host's journal, stamped `service.name` diff --git a/nix/checks.nix b/nix/checks.nix index 1d7a48b0..f04e1888 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -77,6 +77,9 @@ in # Runs ./host-modules/lib/refresh-consumer.nix's restart decision under a # comma-decimal locale, against a stub `systemctl`. script-test-refresh-consumer = import ./script-tests/refresh-consumer.nix { inherit pkgs; }; + # Runs ./host-modules/lib/database-size-push.nix against a symlinked directory, + # as a dynamic user's state directory is: the reported size is the target's. + script-test-database-size-push = import ./script-tests/database-size-push.nix { inherit pkgs; }; # Runs ./host-modules/lib/retry-in-job.nix's `retry_in_job`, which the agent # trust-bundle fetch retries through inside its start job: success after two # failures, the wait schedule, and a return of 1 once the budget is spent. diff --git a/nix/host-modules/hive-matrix.nix b/nix/host-modules/hive-matrix.nix index b164e984..f3e91eda 100644 --- a/nix/host-modules/hive-matrix.nix +++ b/nix/host-modules/hive-matrix.nix @@ -153,6 +153,14 @@ let # Loopback in the container's own netns, and distinct from the other service # containers' forwarders all the same. forwarderTelemetryPort = 8899; + # Loopback in the container's own netns: the forwarder's OTLP/HTTP metrics + # receiver, which the database-size exporter pushes to. + forwarderOtlpPort = 8906; + databaseSizeUnit = "hive-matrix-database-size"; + databaseSizePush = import ./lib/database-size-push.nix { inherit lib; }; + # nixpkgs' default `services.matrix-tuwunel.stateDirectory`, which is also + # the database directory under /var/lib. + stateName = "tuwunel"; # ── the swarm's own appservice ────────────────────────────────────── # @@ -497,6 +505,13 @@ in } ''; + # The collector requests the homeserver through the gateway: tuwunel + # exports no metrics and binds no TCP port, and `httpcheck` dials HTTP(S) + # only, so the container's own forwarder cannot reach the socket. + services.hyperhive.otel.httpChecks = lib.optional ( + cfg.gatewayHost != null + ) "https://${cfg.gatewayHost}/_matrix/client/versions"; + # `server_name = gatewayHost`. `/_matrix/*` → tuwunel (CORS `*`, 50M # body cap, 1h long-poll timeout). `/` serves fluffychat. nginx's # longest-prefix rule puts `/_matrix/` @@ -883,6 +898,7 @@ in clientId = forwarderClientId; tokenUrl = "${toString autheliaUrl}/api/oidc/token"; telemetryPort = forwarderTelemetryPort; + otlpMetricsPort = forwarderOtlpPort; store = { package = deployCfg.bao.package; addr = "https://${baoCfg.domain}:${toString baoCfg.port}"; @@ -893,6 +909,61 @@ in }; }; + # The size of tuwunel's database directory, pushed to the forwarder. + # tuwunel exports no metrics of its own. It runs as tuwunel's dynamic + # user, shared by name, because the state directory is mode 0700; + # with tuwunel stopped the name gets another uid and the read fails. + systemd.services.${databaseSizeUnit} = lib.mkIf ctlActive { + description = "push the size of tuwunel's database directory to the swarm collector"; + wants = [ "tuwunel.service" ]; + after = [ + "tuwunel.service" + "opentelemetry-collector.service" + ]; + path = [ + pkgs.coreutils + pkgs.curl + ]; + serviceConfig = { + Type = "oneshot"; + User = "tuwunel"; + DynamicUser = true; + StateDirectory = stateName; + StateDirectoryMode = "0700"; + SyslogIdentifier = databaseSizeUnit; + UMask = "0077"; + CapabilityBoundingSet = [ "" ]; + NoNewPrivileges = true; + PrivateDevices = true; + PrivateTmp = true; + ProtectSystem = "strict"; + ProtectHome = true; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectControlGroups = true; + RestrictAddressFamilies = [ + "AF_INET" + "AF_INET6" + ]; + RestrictNamespaces = true; + RestrictRealtime = true; + LockPersonality = true; + SystemCallArchitectures = "native"; + }; + script = databaseSizePush { + dir = "/var/lib/${stateName}"; + otlpPort = forwarderOtlpPort; + }; + }; + + systemd.timers.${databaseSizeUnit} = lib.mkIf ctlActive { + wantedBy = [ "timers.target" ]; + timerConfig = { + OnBootSec = "2min"; + OnUnitActiveSec = "5min"; + }; + }; + # Its own netns, so it keeps its own firewall; resolvconf stays on # and serves `networking.nameservers` below. services.hyperhive.swarmContainer = { diff --git a/nix/host-modules/lib/database-size-push.nix b/nix/host-modules/lib/database-size-push.nix new file mode 100644 index 00000000..923c4199 --- /dev/null +++ b/nix/host-modules/lib/database-size-push.nix @@ -0,0 +1,37 @@ +# The hive-matrix container's database-size step: measures a directory and +# pushes the byte count as the gauge `hyperhive.matrix.database_bytes` to the +# forwarder's OTLP/HTTP metrics receiver on loopback. +# +# `dir` is a `DynamicUser` unit's `/var/lib/`, which systemd makes a +# symlink to `/var/lib/private/`. `--dereference-args` measures the +# directory behind the link and works unchanged when `dir` is a real +# directory; without it `du` reports the link itself (a few bytes). +# +# A result under 4096 bytes is not a database, since a directory entry alone +# is that big: the step warns on stderr and pushes nothing, so the gauge +# stays absent rather than wrong. +# +# Pure function — NOT a NixOS module. Needs `du`, `cut`, `date` and `curl`. +{ lib }: +{ dir, otlpPort }: +'' + set -euo pipefail + + bytes="$(du --summarize --bytes --dereference-args ${lib.escapeShellArg dir} | cut -f1)" + case "$bytes" in + "" | *[!0-9]*) + echo "du answered a non-number: bytes='$bytes'" >&2 + exit 1 + ;; + esac + if [ "$bytes" -lt 4096 ]; then + echo "warning: ${dir} measures $bytes bytes, too small for a database; pushing nothing" >&2 + exit 0 + fi + + printf '{"resourceMetrics":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"hive-matrix"}}]},"scopeMetrics":[{"metrics":[{"name":"hyperhive.matrix.database_bytes","gauge":{"dataPoints":[{"asInt":"%s","timeUnixNano":"%s"}]}}]}]}]}' \ + "$bytes" "$(date +%s%N)" | + curl --fail --silent --show-error --max-time 20 \ + -H 'Content-Type: application/json' --data-binary @- \ + http://127.0.0.1:${toString otlpPort}/v1/metrics +'' diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 44864406..9e500061 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -189,6 +189,23 @@ in ''; }; + httpChecks = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + example = [ "https://chat.example/_matrix/client/versions" ]; + description = '' + URLs this hive's collector requests every 60 s with the `httpcheck` + receiver, which reports whether each answered (`httpcheck.status`, + by status class), how long it took (`httpcheck.duration`) and + whether the request failed (`httpcheck.error`). + + **A service adds its own URL, from its own module, under its own + `enable`.** The collector dials from the host, so the name has to + resolve there and its certificate has to chain to the trust bundle + this collector reads. Empty renders no receiver. + ''; + }; + journalUnits = lib.mkOption { type = lib.types.listOf lib.types.str; default = [ ]; @@ -514,6 +531,15 @@ in # validates and starts perfectly while reading nothing — and the # empty set is the default, so that inert shape would be what # most hives deploy. + // lib.optionalAttrs (otel.httpChecks != [ ]) { + httpcheck = { + collection_interval = "60s"; + targets = map (endpoint: { + inherit endpoint; + method = "GET"; + }) otel.httpChecks; + }; + } // lib.optionalAttrs (otel.scrapeTargets != { }) { prometheus.config.scrape_configs = lib.mapAttrsToList (job: target: { job_name = job; @@ -638,6 +664,7 @@ in "hostmetrics" ] ++ lib.optional (otel.scrapeTargets != { }) "prometheus" + ++ lib.optional (otel.httpChecks != [ ]) "httpcheck" ++ lib.optional gatewayCfg.enable "nginx"; # Applies to EVERY receiver above, not just `hostmetrics` — a # processor sits in the pipeline, not on a receiver. That is diff --git a/nix/module-eval/matrix-core.nix b/nix/module-eval/matrix-core.nix index 6302ba0a..8bea9c56 100644 --- a/nix/module-eval/matrix-core.nix +++ b/nix/module-eval/matrix-core.nix @@ -178,6 +178,65 @@ let && !(apex ? "= /.well-known/matrix/client") && !(apex ? "= /.well-known/matrix/server"); } + { + # tuwunel has no metrics of its own, so the hive collector requests + # `/_matrix/client/versions` through the gateway. The URL carries no + # port, like the rest of this module's URLs. The elsewhere host is the + # control: the same lookups there find no check, and the receiver stays + # out of its pipeline. + name = "the hive collector requests the homeserver's versions endpoint through the gateway, only where the homeserver runs"; + ok = + let + url = "https://chat.t.local/_matrix/client/versions"; + settings = m: m.services.opentelemetry-collector.settings; + pipeline = m: (settings m).service.pipelines.metrics.receivers; + in + matrixNoBaoIdentity.services.hyperhive.otel.httpChecks == [ url ] + && + (settings matrixNoBaoIdentity).receivers.httpcheck.targets == [ + { + endpoint = url; + method = "GET"; + } + ] + && lib.elem "httpcheck" (pipeline matrixNoBaoIdentity) + && matrixElsewhere.services.hyperhive.otel.httpChecks == [ ] + && !((settings matrixElsewhere).receivers ? httpcheck) + && !(lib.elem "httpcheck" (pipeline matrixElsewhere)) + # Presence control for the two lookups above. + && (settings matrixElsewhere).receivers ? hostmetrics + && lib.elem "hostmetrics" (pipeline matrixElsewhere); + } + { + # The size is read from the database directory, as the dynamic user the + # homeserver runs as, and pushed to the forwarder's OTLP port. Without a + # store identity the container has no forwarder, so there is nothing to + # push to and neither unit appears; the hive with the identity is the + # control that the same lookups find them. + name = "the homeserver container pushes its database size through the forwarder, and only with a forwarder"; + ok = + let + inner = m: m.containers.hive-matrix.config; + here = inner matrixBodyCap; + without = inner matrixNoBaoIdentity; + svc = here.systemd.services.hive-matrix-database-size; + f = here.services.hyperhive.swarmJournalForwarder; + in + svc.serviceConfig.Type == "oneshot" + && svc.serviceConfig.DynamicUser + && svc.serviceConfig.User == "tuwunel" + && svc.serviceConfig.StateDirectory == here.services.matrix-tuwunel.stateDirectory + && lib.hasInfix "du --summarize --bytes --dereference-args /var/lib/tuwunel" svc.script + && lib.hasInfix "-lt 4096" svc.script + && lib.hasInfix "hyperhive.matrix.database_bytes" svc.script + && lib.hasInfix "127.0.0.1:${toString f.otlpMetricsPort}/v1/metrics" svc.script + && f.otlpMetricsPort != null + && here.systemd.timers.hive-matrix-database-size.timerConfig.OnUnitActiveSec == "5min" + && !(without.systemd.services ? hive-matrix-database-size) + && !(without.systemd.timers ? hive-matrix-database-size) + # Presence control: the container's units are found by the same lookup. + && without.systemd.services ? tuwunel; + } ]; in runGroup "matrix-core" cases diff --git a/nix/script-tests/database-size-push.nix b/nix/script-tests/database-size-push.nix new file mode 100644 index 00000000..4f703956 --- /dev/null +++ b/nix/script-tests/database-size-push.nix @@ -0,0 +1,21 @@ +# `checks.script-test-database-size-push` — runs ../host-modules/lib/database-size-push.nix's +# script against a symlinked directory, as a `DynamicUser` unit's state directory +# is, with a stub `curl`. The cases are in ./database-size-push.sh. +{ + pkgs, + ... +}: +pkgs.runCommand "hyperhive-script-test-database-size-push" + { + nativeBuildInputs = [ pkgs.coreutils ]; + SCRIPT_LINK = pkgs.writeText "database-size-push-link.sh" ( + import ../host-modules/lib/database-size-push.nix { inherit (pkgs) lib; } { + dir = "@DIR@"; + otlpPort = 8906; + } + ); + } + '' + ${pkgs.bash}/bin/bash ${./database-size-push.sh} + touch "$out" + '' diff --git a/nix/script-tests/database-size-push.sh b/nix/script-tests/database-size-push.sh new file mode 100644 index 00000000..bd0f6eab --- /dev/null +++ b/nix/script-tests/database-size-push.sh @@ -0,0 +1,74 @@ +# Cases for ./database-size-push.nix. The directory is reached through a +# symlink, as a dynamic user's state directory is. `curl` is a stub that +# records the payload and the URL it is given. +set -uo pipefail + +failures=0 + +fail() { + echo "FAILED: $case: $*" >&2 + failures=$((failures + 1)) +} + +root=$(mktemp -d) +mkdir -p "$root/bin" "$root/private/tuwunel" +head -c 100000 /dev/zero >"$root/private/tuwunel/db" +ln -s "$root/private/tuwunel" "$root/tuwunel" + +cat >"$root/bin/curl" <"$root/curl.args" +cat >"$root/curl.payload" +STUB +chmod +x "$root/bin/curl" + +# Runs the script for \$1, returns its status; stdout and stderr land in files. +run() { + rm -f "$root/curl.args" "$root/curl.payload" + sed "s|@DIR@|$1|" "$SCRIPT_LINK" >"$root/script.sh" + PATH="$root/bin:$PATH" bash "$root/script.sh" >"$root/out" 2>"$root/err" +} + +case="a symlinked directory reports the target's size" +want=$(du --summarize --bytes "$root/private/tuwunel/" | cut -f1) +link=$(du --summarize --bytes "$root/tuwunel" | cut -f1) +[ "$want" -ge 100000 ] || fail "fixture target is $want bytes" +[ "$link" -lt 4096 ] || fail "fixture link measures $link bytes, so it does not tell the cases apart" +if run "$root/tuwunel"; then + grep -q "\"asInt\":\"$want\"" "$root/curl.payload" || fail "payload is $(cat "$root/curl.payload"), want asInt $want" + grep -q 'hyperhive.matrix.database_bytes' "$root/curl.payload" || fail "no metric name in the payload" + grep -q '127.0.0.1:8906/v1/metrics' "$root/curl.args" || fail "wrong url: $(cat "$root/curl.args")" +else + fail "the script failed: $(cat "$root/err")" +fi + +case="a real directory reports its own size" +if run "$root/private/tuwunel"; then + grep -q "\"asInt\":\"$want\"" "$root/curl.payload" || fail "payload is $(cat "$root/curl.payload"), want asInt $want" +else + fail "the script failed: $(cat "$root/err")" +fi + +# `du` stub answering a link-sized number, as an old command line would. +case="an implausibly small size warns and pushes nothing" +cat >"$root/bin/du" <