matrix: report homeserver up, latency and database size through existing collectors
tuwunel 1.9.4 exports no metrics. The host collector gets an httpcheck receiver on the gateway's /_matrix/client/versions (new services.hyperhive.otel.httpChecks, declared by hive-matrix), and a timer in the hive-matrix container pushes the database directory's size through the forwarder's OTLP port, as the attic cache-size unit does. The size step dereferences the state directory's symlink and pushes nothing for a result under 4096 bytes.
This commit is contained in:
atlas
mara
parent
839dbd642a
commit
bef9d401a4
3 changed files with 311 additions and 0 deletions
|
|
@ -153,6 +153,14 @@ let
|
|||
# Loopback in the container's own netns, and distinct from the other service
|
||||
# containers' forwarders all the same.
|
||||
forwarderTelemetryPort = 8899;
|
||||
# Loopback in the container's own netns: the forwarder's OTLP/HTTP metrics
|
||||
# receiver, which the database-size exporter pushes to.
|
||||
forwarderOtlpPort = 8906;
|
||||
databaseSizeUnit = "hive-matrix-database-size";
|
||||
databaseSizePush = import ./lib/database-size-push.nix { inherit lib; };
|
||||
# nixpkgs' default `services.matrix-tuwunel.stateDirectory`, which is also
|
||||
# the database directory under /var/lib.
|
||||
stateName = "tuwunel";
|
||||
|
||||
# ── the swarm's own appservice ──────────────────────────────────────
|
||||
#
|
||||
|
|
@ -497,6 +505,13 @@ in
|
|||
}
|
||||
'';
|
||||
|
||||
# The collector requests the homeserver through the gateway: tuwunel
|
||||
# exports no metrics and binds no TCP port, and `httpcheck` dials HTTP(S)
|
||||
# only, so the container's own forwarder cannot reach the socket.
|
||||
services.hyperhive.otel.httpChecks = lib.optional (
|
||||
cfg.gatewayHost != null
|
||||
) "https://${cfg.gatewayHost}/_matrix/client/versions";
|
||||
|
||||
# `server_name = gatewayHost`. `/_matrix/*` → tuwunel (CORS `*`, 50M
|
||||
# body cap, 1h long-poll timeout). `/` serves fluffychat. nginx's
|
||||
# longest-prefix rule puts `/_matrix/`
|
||||
|
|
@ -883,6 +898,7 @@ in
|
|||
clientId = forwarderClientId;
|
||||
tokenUrl = "${toString autheliaUrl}/api/oidc/token";
|
||||
telemetryPort = forwarderTelemetryPort;
|
||||
otlpMetricsPort = forwarderOtlpPort;
|
||||
store = {
|
||||
package = deployCfg.bao.package;
|
||||
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
|
|
@ -893,6 +909,61 @@ in
|
|||
};
|
||||
};
|
||||
|
||||
# The size of tuwunel's database directory, pushed to the forwarder.
|
||||
# tuwunel exports no metrics of its own. It runs as tuwunel's dynamic
|
||||
# user, shared by name, because the state directory is mode 0700;
|
||||
# with tuwunel stopped the name gets another uid and the read fails.
|
||||
systemd.services.${databaseSizeUnit} = lib.mkIf ctlActive {
|
||||
description = "push the size of tuwunel's database directory to the swarm collector";
|
||||
wants = [ "tuwunel.service" ];
|
||||
after = [
|
||||
"tuwunel.service"
|
||||
"opentelemetry-collector.service"
|
||||
];
|
||||
path = [
|
||||
pkgs.coreutils
|
||||
pkgs.curl
|
||||
];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
User = "tuwunel";
|
||||
DynamicUser = true;
|
||||
StateDirectory = stateName;
|
||||
StateDirectoryMode = "0700";
|
||||
SyslogIdentifier = databaseSizeUnit;
|
||||
UMask = "0077";
|
||||
CapabilityBoundingSet = [ "" ];
|
||||
NoNewPrivileges = true;
|
||||
PrivateDevices = true;
|
||||
PrivateTmp = true;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
ProtectKernelTunables = true;
|
||||
ProtectKernelModules = true;
|
||||
ProtectControlGroups = true;
|
||||
RestrictAddressFamilies = [
|
||||
"AF_INET"
|
||||
"AF_INET6"
|
||||
];
|
||||
RestrictNamespaces = true;
|
||||
RestrictRealtime = true;
|
||||
LockPersonality = true;
|
||||
SystemCallArchitectures = "native";
|
||||
};
|
||||
script = databaseSizePush {
|
||||
dir = "/var/lib/${stateName}";
|
||||
otlpPort = forwarderOtlpPort;
|
||||
};
|
||||
};
|
||||
|
||||
systemd.timers.${databaseSizeUnit} = lib.mkIf ctlActive {
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnBootSec = "2min";
|
||||
OnUnitActiveSec = "5min";
|
||||
};
|
||||
};
|
||||
|
||||
# Its own netns, so it keeps its own firewall; resolvconf stays on
|
||||
# and serves `networking.nameservers` below.
|
||||
services.hyperhive.swarmContainer = {
|
||||
|
|
|
|||
37
nix/host-modules/lib/database-size-push.nix
Normal file
37
nix/host-modules/lib/database-size-push.nix
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
# The hive-matrix container's database-size step: measures a directory and
|
||||
# pushes the byte count as the gauge `hyperhive.matrix.database_bytes` to the
|
||||
# forwarder's OTLP/HTTP metrics receiver on loopback.
|
||||
#
|
||||
# `dir` is a `DynamicUser` unit's `/var/lib/<name>`, which systemd makes a
|
||||
# symlink to `/var/lib/private/<name>`. `--dereference-args` measures the
|
||||
# directory behind the link and works unchanged when `dir` is a real
|
||||
# directory; without it `du` reports the link itself (a few bytes).
|
||||
#
|
||||
# A result under 4096 bytes is not a database, since a directory entry alone
|
||||
# is that big: the step warns on stderr and pushes nothing, so the gauge
|
||||
# stays absent rather than wrong.
|
||||
#
|
||||
# Pure function — NOT a NixOS module. Needs `du`, `cut`, `date` and `curl`.
|
||||
{ lib }:
|
||||
{ dir, otlpPort }:
|
||||
''
|
||||
set -euo pipefail
|
||||
|
||||
bytes="$(du --summarize --bytes --dereference-args ${lib.escapeShellArg dir} | cut -f1)"
|
||||
case "$bytes" in
|
||||
"" | *[!0-9]*)
|
||||
echo "du answered a non-number: bytes='$bytes'" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
if [ "$bytes" -lt 4096 ]; then
|
||||
echo "warning: ${dir} measures $bytes bytes, too small for a database; pushing nothing" >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
printf '{"resourceMetrics":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"hive-matrix"}}]},"scopeMetrics":[{"metrics":[{"name":"hyperhive.matrix.database_bytes","gauge":{"dataPoints":[{"asInt":"%s","timeUnixNano":"%s"}]}}]}]}]}' \
|
||||
"$bytes" "$(date +%s%N)" |
|
||||
curl --fail --silent --show-error --max-time 20 \
|
||||
-H 'Content-Type: application/json' --data-binary @- \
|
||||
http://127.0.0.1:${toString otlpPort}/v1/metrics
|
||||
''
|
||||
|
|
@ -189,6 +189,23 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
httpChecks = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
example = [ "https://chat.example/_matrix/client/versions" ];
|
||||
description = ''
|
||||
URLs this hive's collector requests every 60 s with the `httpcheck`
|
||||
receiver, which reports whether each answered (`httpcheck.status`,
|
||||
by status class), how long it took (`httpcheck.duration`) and
|
||||
whether the request failed (`httpcheck.error`).
|
||||
|
||||
**A service adds its own URL, from its own module, under its own
|
||||
`enable`.** The collector dials from the host, so the name has to
|
||||
resolve there and its certificate has to chain to the trust bundle
|
||||
this collector reads. Empty renders no receiver.
|
||||
'';
|
||||
};
|
||||
|
||||
journalUnits = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
|
|
@ -514,6 +531,15 @@ in
|
|||
# validates and starts perfectly while reading nothing — and the
|
||||
# empty set is the default, so that inert shape would be what
|
||||
# most hives deploy.
|
||||
// lib.optionalAttrs (otel.httpChecks != [ ]) {
|
||||
httpcheck = {
|
||||
collection_interval = "60s";
|
||||
targets = map (endpoint: {
|
||||
inherit endpoint;
|
||||
method = "GET";
|
||||
}) otel.httpChecks;
|
||||
};
|
||||
}
|
||||
// lib.optionalAttrs (otel.scrapeTargets != { }) {
|
||||
prometheus.config.scrape_configs = lib.mapAttrsToList (job: target: {
|
||||
job_name = job;
|
||||
|
|
@ -638,6 +664,7 @@ in
|
|||
"hostmetrics"
|
||||
]
|
||||
++ lib.optional (otel.scrapeTargets != { }) "prometheus"
|
||||
++ lib.optional (otel.httpChecks != [ ]) "httpcheck"
|
||||
++ lib.optional gatewayCfg.enable "nginx";
|
||||
# Applies to EVERY receiver above, not just `hostmetrics` — a
|
||||
# processor sits in the pipeline, not on a receiver. That is
|
||||
|
|
|
|||
Loading…
Reference in a new issue