diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index 7e982e48..9d33467f 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -28,6 +28,7 @@ ./swarm-controller.nix ./swarm-snapshot-store.nix ./swarm-ui.nix + ./swarm-victoriametrics.nix ./swarm-wireguard.nix ./swarm.nix ./swarm-peers-removed.nix diff --git a/nix/host-modules/swarm-victoriametrics.nix b/nix/host-modules/swarm-victoriametrics.nix new file mode 100644 index 00000000..c93b7cd7 --- /dev/null +++ b/nix/host-modules/swarm-victoriametrics.nix @@ -0,0 +1,171 @@ +# The swarm's metrics store: one VictoriaMetrics for the whole swarm, in a +# `swarm-victoriametrics` nixos-container. +# +# A LOCAL time-series database rather than only an external sink, and that is +# the point rather than a convenience: the swarm dashboard has to stay +# readable when the outside world is unreachable. Same failure-domain property +# the hive-status KV has — a view of the system must not depend on the system +# it is viewing being healthy. +# +# It is the collector that feeds this (the gateway OTEL collector), not the +# agents directly: one ingest point per swarm, authenticated there. +{ + pkgs, + lib, + config, + ... +}: +let + cfg = config.services.hyperhive.swarm.victoriametrics; + hyperhiveCfg = config.services.hyperhive; + gatewayCfg = hyperhiveCfg.gateway; + swarmDomain = hyperhiveCfg.swarm.domain; + + # Total on a null swarm domain for the same reason every sibling module is: + # the required-domain assertion in hive-network.nix should be what an + # operator sees, not a coercion error from here. + domainBase = if swarmDomain == null then "invalid" else swarmDomain; +in +{ + options.services.hyperhive.swarm.victoriametrics = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Run the swarm's metrics store on this host. Off by default and not + derived from {option}`services.hyperhive.enable`: a swarm has one + metrics store, so enabling it is a decision about swarm topology + rather than about whether hyperhive is installed. + ''; + }; + + package = lib.mkOption { + type = lib.types.package; + default = pkgs.victoriametrics; + defaultText = lib.literalExpression "pkgs.victoriametrics"; + description = "VictoriaMetrics package to run."; + }; + + machine = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = "swarm-victoriametrics"; + description = '' + Container name. Read-only: the name appears in host paths and in + `machinectl`, so it is a fact other modules may read rather than a + knob. + ''; + }; + + domain = lib.mkOption { + type = lib.types.str; + default = "metrics.${domainBase}"; + defaultText = lib.literalExpression ''"metrics.''${services.hyperhive.swarm.domain}"''; + description = '' + Name the gateway serves this on. A sibling of the swarm's other + service names, so the swarm-services sub-CA can issue for it — see + `hive-tls.nix` for why a service name being a sibling rather than a + child decides which CA may sign it. + ''; + }; + + port = lib.mkOption { + type = lib.types.port; + default = 8428; + description = '' + Port VictoriaMetrics listens on, bound to loopback only (see + below). Upstream's own default, kept so an operator reading + VictoriaMetrics documentation finds what they expect. + ''; + }; + + retentionPeriod = lib.mkOption { + type = lib.types.str; + default = "5y"; + example = "90d"; + description = '' + How long samples are kept. + + Deliberately a high default rather than a required option: the two + failure directions are not symmetric. Too long fills a disk, which + is visible and recoverable by lowering this; too short **destroys + history**, silently and permanently. So the safe default is generous + and an operator lowers it once they have measured how fast this swarm + actually accumulates data. + ''; + }; + }; + + config = lib.mkIf (hyperhiveCfg.enable && cfg.enable) { + # The gateway name and the quick-link, both inside `cfg.enable` — that + # guard is the load-bearing part. Every hive in a swarm may know this + # store exists, but only the host that RUNS it may claim the name; a + # client hive declaring the vhost would answer for a service it does not + # have. + services.hyperhive.gateway.localNames = [ cfg.domain ]; + + services.hyperhive.swarm.controller.links = [ + { + label = "Metrics"; + icon = "📈"; + url = "https://${cfg.domain}/"; + } + ]; + + services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // { + listen = gatewayCfg.lib.listen; + extraConfig = gatewayCfg.lib.securityHeaders; + locations."/" = { + proxyPass = "http://127.0.0.1:${toString cfg.port}/"; + }; + }; + + containers.${cfg.machine} = { + autoStart = true; + ephemeral = false; + # Shared host netns, like every sibling swarm container: the gateway + # reaches this at 127.0.0.1:. + privateNetwork = false; + + config = + { ... }: + { + system.stateVersion = "26.05"; + + # This container shares the host netns, so its own firewall.service + # would rewrite the HOST ruleset at every boot. The host firewall + # owns all filtering. + networking.firewall.enable = false; + # Keep the host-copied /etc/resolv.conf intact — resolvconf's + # host-tracking would regenerate it to an empty file, since the + # host's copy doesn't cross the boundary after start. + networking.resolvconf.enable = lib.mkForce false; + + services.victoriametrics = { + enable = true; + package = cfg.package; + retentionPeriod = cfg.retentionPeriod; + + # ⚠️ PINNED TO LOOPBACK, and this is a correction rather than a + # preference: upstream's default is `:8428`, i.e. every + # interface. The gateway is the only intended client and it is on + # this host, so binding wider would publish an unauthenticated + # write endpoint (see the OTLP note below) to whatever the host + # is reachable on. + listenAddress = "127.0.0.1:${toString cfg.port}"; + }; + }; + }; + }; + + # 🔑 OTLP ingest needs no flag. Measured against the pinned 1.146.0 rather + # than inferred from the module's option list, which has no OTLP switch and + # so reads as though the feature were absent: the running server answers + # `POST /opentelemetry/api/v1/push` with 200 (a nonexistent path answers + # 400, so that 200 means the route exists). The `-opentelemetry.*` flags + # only tune naming and limits, and are reachable via `extraOptions` if a + # deployment ever needs them. + # + # ⚠️ That endpoint is unauthenticated, which is why `listenAddress` above is + # loopback and why the collector — not agents — is the writer. +}