From bedfa786a79ff609baf39e06c35dd0e2c5801191 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 16 Aug 2026 21:25:04 +0200 Subject: [PATCH] feat(#3265): swarm metrics store as a VictoriaMetrics container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A local time-series database rather than only an external sink, so the swarm dashboard stays readable when the outside world is not: a view of the system must not depend on the system it views being healthy. listenAddress is pinned to loopback. Upstream defaults it to every interface, and the OTLP ingest path this exists to receive on is unauthenticated — the gateway is the only intended client and it is on this host, so a wider bind would publish a write endpoint to whatever the host is reachable on. retentionPeriod defaults high rather than being required, because the two failure directions are not symmetric: too long fills a disk, which is visible and recoverable by lowering it, while too short destroys history silently and permanently. The operator lowers it once they have measured how fast this swarm accumulates. OTLP needs no flag. Measured against the pinned 1.146.0 rather than inferred from the module's option list, which has no OTLP switch and so reads as though the feature were missing: the running server answers POST /opentelemetry/api/v1/push with 200, where a nonexistent path answers 400. --- nix/host-modules/default.nix | 1 + nix/host-modules/swarm-victoriametrics.nix | 171 +++++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 nix/host-modules/swarm-victoriametrics.nix diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index 7e982e48..9d33467f 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -28,6 +28,7 @@ ./swarm-controller.nix ./swarm-snapshot-store.nix ./swarm-ui.nix + ./swarm-victoriametrics.nix ./swarm-wireguard.nix ./swarm.nix ./swarm-peers-removed.nix diff --git a/nix/host-modules/swarm-victoriametrics.nix b/nix/host-modules/swarm-victoriametrics.nix new file mode 100644 index 00000000..c93b7cd7 --- /dev/null +++ b/nix/host-modules/swarm-victoriametrics.nix @@ -0,0 +1,171 @@ +# The swarm's metrics store: one VictoriaMetrics for the whole swarm, in a +# `swarm-victoriametrics` nixos-container. +# +# A LOCAL time-series database rather than only an external sink, and that is +# the point rather than a convenience: the swarm dashboard has to stay +# readable when the outside world is unreachable. Same failure-domain property +# the hive-status KV has — a view of the system must not depend on the system +# it is viewing being healthy. +# +# It is the collector that feeds this (the gateway OTEL collector), not the +# agents directly: one ingest point per swarm, authenticated there. +{ + pkgs, + lib, + config, + ... +}: +let + cfg = config.services.hyperhive.swarm.victoriametrics; + hyperhiveCfg = config.services.hyperhive; + gatewayCfg = hyperhiveCfg.gateway; + swarmDomain = hyperhiveCfg.swarm.domain; + + # Total on a null swarm domain for the same reason every sibling module is: + # the required-domain assertion in hive-network.nix should be what an + # operator sees, not a coercion error from here. + domainBase = if swarmDomain == null then "invalid" else swarmDomain; +in +{ + options.services.hyperhive.swarm.victoriametrics = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Run the swarm's metrics store on this host. Off by default and not + derived from {option}`services.hyperhive.enable`: a swarm has one + metrics store, so enabling it is a decision about swarm topology + rather than about whether hyperhive is installed. + ''; + }; + + package = lib.mkOption { + type = lib.types.package; + default = pkgs.victoriametrics; + defaultText = lib.literalExpression "pkgs.victoriametrics"; + description = "VictoriaMetrics package to run."; + }; + + machine = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = "swarm-victoriametrics"; + description = '' + Container name. Read-only: the name appears in host paths and in + `machinectl`, so it is a fact other modules may read rather than a + knob. + ''; + }; + + domain = lib.mkOption { + type = lib.types.str; + default = "metrics.${domainBase}"; + defaultText = lib.literalExpression ''"metrics.''${services.hyperhive.swarm.domain}"''; + description = '' + Name the gateway serves this on. A sibling of the swarm's other + service names, so the swarm-services sub-CA can issue for it — see + `hive-tls.nix` for why a service name being a sibling rather than a + child decides which CA may sign it. + ''; + }; + + port = lib.mkOption { + type = lib.types.port; + default = 8428; + description = '' + Port VictoriaMetrics listens on, bound to loopback only (see + below). Upstream's own default, kept so an operator reading + VictoriaMetrics documentation finds what they expect. + ''; + }; + + retentionPeriod = lib.mkOption { + type = lib.types.str; + default = "5y"; + example = "90d"; + description = '' + How long samples are kept. + + Deliberately a high default rather than a required option: the two + failure directions are not symmetric. Too long fills a disk, which + is visible and recoverable by lowering this; too short **destroys + history**, silently and permanently. So the safe default is generous + and an operator lowers it once they have measured how fast this swarm + actually accumulates data. + ''; + }; + }; + + config = lib.mkIf (hyperhiveCfg.enable && cfg.enable) { + # The gateway name and the quick-link, both inside `cfg.enable` — that + # guard is the load-bearing part. Every hive in a swarm may know this + # store exists, but only the host that RUNS it may claim the name; a + # client hive declaring the vhost would answer for a service it does not + # have. + services.hyperhive.gateway.localNames = [ cfg.domain ]; + + services.hyperhive.swarm.controller.links = [ + { + label = "Metrics"; + icon = "📈"; + url = "https://${cfg.domain}/"; + } + ]; + + services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // { + listen = gatewayCfg.lib.listen; + extraConfig = gatewayCfg.lib.securityHeaders; + locations."/" = { + proxyPass = "http://127.0.0.1:${toString cfg.port}/"; + }; + }; + + containers.${cfg.machine} = { + autoStart = true; + ephemeral = false; + # Shared host netns, like every sibling swarm container: the gateway + # reaches this at 127.0.0.1:. + privateNetwork = false; + + config = + { ... }: + { + system.stateVersion = "26.05"; + + # This container shares the host netns, so its own firewall.service + # would rewrite the HOST ruleset at every boot. The host firewall + # owns all filtering. + networking.firewall.enable = false; + # Keep the host-copied /etc/resolv.conf intact — resolvconf's + # host-tracking would regenerate it to an empty file, since the + # host's copy doesn't cross the boundary after start. + networking.resolvconf.enable = lib.mkForce false; + + services.victoriametrics = { + enable = true; + package = cfg.package; + retentionPeriod = cfg.retentionPeriod; + + # ⚠️ PINNED TO LOOPBACK, and this is a correction rather than a + # preference: upstream's default is `:8428`, i.e. every + # interface. The gateway is the only intended client and it is on + # this host, so binding wider would publish an unauthenticated + # write endpoint (see the OTLP note below) to whatever the host + # is reachable on. + listenAddress = "127.0.0.1:${toString cfg.port}"; + }; + }; + }; + }; + + # 🔑 OTLP ingest needs no flag. Measured against the pinned 1.146.0 rather + # than inferred from the module's option list, which has no OTLP switch and + # so reads as though the feature were absent: the running server answers + # `POST /opentelemetry/api/v1/push` with 200 (a nonexistent path answers + # 400, so that 200 means the route exists). The `-opentelemetry.*` flags + # only tune naming and limits, and are reachable via `extraOptions` if a + # deployment ever needs them. + # + # ⚠️ That endpoint is unauthenticated, which is why `listenAddress` above is + # loopback and why the collector — not agents — is the writer. +}