vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(#3265): swarm metrics store as a VictoriaMetrics container

Browse source 
A local time-series database rather than only an external sink, so the
swarm dashboard stays readable when the outside world is not: a view of
the system must not depend on the system it views being healthy.

listenAddress is pinned to loopback. Upstream defaults it to every
interface, and the OTLP ingest path this exists to receive on is
unauthenticated — the gateway is the only intended client and it is on
this host, so a wider bind would publish a write endpoint to whatever
the host is reachable on.

retentionPeriod defaults high rather than being required, because the
two failure directions are not symmetric: too long fills a disk, which
is visible and recoverable by lowering it, while too short destroys
history silently and permanently. The operator lowers it once they have
measured how fast this swarm accumulates.

OTLP needs no flag. Measured against the pinned 1.146.0 rather than
inferred from the module's option list, which has no OTLP switch and so
reads as though the feature were missing: the running server answers
POST /opentelemetry/api/v1/push with 200, where a nonexistent path
answers 400.
This commit is contained in:
atlas 2026-08-16 21:25:04 +02:00 • committed by mara
commit bedfa786a7
2 changed files with 172 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
nix/host-modules/default.nix
View file

@ -28,6 +28,7 @@
./swarm-controller.nix
./swarm-snapshot-store.nix
./swarm-ui.nix
./swarm-victoriametrics.nix
./swarm-wireguard.nix
./swarm.nix
./swarm-peers-removed.nix

171
nix/host-modules/swarm-victoriametrics.nix Normal file
View file

@ -0,0 +1,171 @@
# The swarm's metrics store: one VictoriaMetrics for the whole swarm, in a
# `swarm-victoriametrics` nixos-container.
#
# A LOCAL time-series database rather than only an external sink, and that is
# the point rather than a convenience: the swarm dashboard has to stay
# readable when the outside world is unreachable. Same failure-domain property
# the hive-status KV has — a view of the system must not depend on the system
# it is viewing being healthy.
#
# It is the collector that feeds this (the gateway OTEL collector), not the
# agents directly: one ingest point per swarm, authenticated there.
{
pkgs,
lib,
config,
...
}:
let
cfg = config.services.hyperhive.swarm.victoriametrics;
hyperhiveCfg = config.services.hyperhive;
gatewayCfg = hyperhiveCfg.gateway;
swarmDomain = hyperhiveCfg.swarm.domain;
# Total on a null swarm domain for the same reason every sibling module is:
# the required-domain assertion in hive-network.nix should be what an
# operator sees, not a coercion error from here.
domainBase = if swarmDomain == null then "invalid" else swarmDomain;
in
{
options.services.hyperhive.swarm.victoriametrics = {
enable = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Run the swarm's metrics store on this host. Off by default and not
derived from {option}`services.hyperhive.enable`: a swarm has one
metrics store, so enabling it is a decision about swarm topology
rather than about whether hyperhive is installed.
'';
};
package = lib.mkOption {
type = lib.types.package;
default = pkgs.victoriametrics;
defaultText = lib.literalExpression "pkgs.victoriametrics";
description = "VictoriaMetrics package to run.";
};
machine = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "swarm-victoriametrics";
description = ''
Container name. Read-only: the name appears in host paths and in
`machinectl`, so it is a fact other modules may read rather than a
knob.
'';
};
domain = lib.mkOption {
type = lib.types.str;
default = "metrics.${domainBase}";
defaultText = lib.literalExpression ''"metrics.''${services.hyperhive.swarm.domain}"'';
description = ''
Name the gateway serves this on. A sibling of the swarm's other
service names, so the swarm-services sub-CA can issue for it — see
`hive-tls.nix` for why a service name being a sibling rather than a
child decides which CA may sign it.
'';
};
port = lib.mkOption {
type = lib.types.port;
default = 8428;
description = ''
Port VictoriaMetrics listens on, bound to loopback only (see
below). Upstream's own default, kept so an operator reading
VictoriaMetrics documentation finds what they expect.
'';
};
retentionPeriod = lib.mkOption {
type = lib.types.str;
default = "5y";
example = "90d";
description = ''
How long samples are kept.
Deliberately a high default rather than a required option: the two
failure directions are not symmetric. Too long fills a disk, which
is visible and recoverable by lowering this; too short **destroys
history**, silently and permanently. So the safe default is generous
and an operator lowers it once they have measured how fast this swarm
actually accumulates data.
'';
};
};
config = lib.mkIf (hyperhiveCfg.enable && cfg.enable) {
# The gateway name and the quick-link, both inside `cfg.enable` — that
# guard is the load-bearing part. Every hive in a swarm may know this
# store exists, but only the host that RUNS it may claim the name; a
# client hive declaring the vhost would answer for a service it does not
# have.
services.hyperhive.gateway.localNames = [ cfg.domain ];
services.hyperhive.swarm.controller.links = [
{
label = "Metrics";
icon = "📈";
url = "https://${cfg.domain}/";
}
];
services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // {
listen = gatewayCfg.lib.listen;
extraConfig = gatewayCfg.lib.securityHeaders;
locations."/" = {
proxyPass = "http://127.0.0.1:${toString cfg.port}/";
};
};
containers.${cfg.machine} = {
autoStart = true;
ephemeral = false;
# Shared host netns, like every sibling swarm container: the gateway
# reaches this at 127.0.0.1:<port>.
privateNetwork = false;
config =
{ ... }:
{
system.stateVersion = "26.05";
# This container shares the host netns, so its own firewall.service
# would rewrite the HOST ruleset at every boot. The host firewall
# owns all filtering.
networking.firewall.enable = false;
# Keep the host-copied /etc/resolv.conf intact — resolvconf's
# host-tracking would regenerate it to an empty file, since the
# host's copy doesn't cross the boundary after start.
networking.resolvconf.enable = lib.mkForce false;
services.victoriametrics = {
enable = true;
package = cfg.package;
retentionPeriod = cfg.retentionPeriod;
# ⚠️ PINNED TO LOOPBACK, and this is a correction rather than a
# preference: upstream's default is `:8428`, i.e. every
# interface. The gateway is the only intended client and it is on
# this host, so binding wider would publish an unauthenticated
# write endpoint (see the OTLP note below) to whatever the host
# is reachable on.
listenAddress = "127.0.0.1:${toString cfg.port}";
};
};
};
};
# 🔑 OTLP ingest needs no flag. Measured against the pinned 1.146.0 rather
# than inferred from the module's option list, which has no OTLP switch and
# so reads as though the feature were absent: the running server answers
# `POST /opentelemetry/api/v1/push` with 200 (a nonexistent path answers
# 400, so that 200 means the route exists). The `-opentelemetry.*` flags
# only tune naming and limits, and are reachable via `extraOptions` if a
# deployment ever needs them.
#
# ⚠️ That endpoint is unauthenticated, which is why `listenAddress` above is
# loopback and why the collector — not agents — is the writer.
}