From 527e07c5e0e8ebe9419010292a53485e6c65b7aa Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 24 Aug 2026 16:36:18 +0200 Subject: [PATCH] feat(swarm-victorialogs): a log store for the swarm An agent can verify that a unit was launched and never that it is working: container journals are not reachable, so a diagnosis stops at the first broken component -- which is precisely the component whose own instrument is least likely to be legible. This is the store half of collecting logs centrally so the question becomes answerable. Mirrors swarm-victoriametrics deliberately: same container shape, same loopback pin, same self-scrape arrangement. Two differences, both intentional. No gateway vhost. VictoriaLogs' ingest and query endpoints carry no authentication of their own, exactly like the metrics store's -- and the metrics store IS published under a resolvable name, which is an open question rather than a settled design. Publishing this one the same way would repeat that before the first instance is decided. Retention defaults to 30d against the metrics store's 5y. Logs are orders of magnitude larger per unit of time and their value decays much faster: a log line answers what happened during an incident, a metric answers whether this is worse than last quarter. Not wired into enableRequiredServices yet -- that lands with the collector pipeline, so we do not start a store nothing writes to. --- nix/host-modules/default.nix | 1 + nix/host-modules/swarm-victorialogs.nix | 168 ++++++++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 nix/host-modules/swarm-victorialogs.nix diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index 666490b5..da0225c9 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -30,6 +30,7 @@ ./swarm-otel.nix ./swarm-snapshot-store.nix ./swarm-ui.nix + ./swarm-victorialogs.nix ./swarm-victoriametrics.nix ./swarm-wireguard.nix ./swarm.nix diff --git a/nix/host-modules/swarm-victorialogs.nix b/nix/host-modules/swarm-victorialogs.nix new file mode 100644 index 00000000..3e1e4781 --- /dev/null +++ b/nix/host-modules/swarm-victorialogs.nix @@ -0,0 +1,168 @@ +# The swarm's log store: one VictoriaLogs for the whole swarm, in a +# `swarm-victorialogs` nixos-container beside the metrics store it mirrors. +# +# Why a store at all rather than reading journals directly: an agent can +# verify that a unit was *launched* and never that it is *working*. The +# container journals are not reachable from an agent, so a diagnosis stops at +# the first component that is broken — which is precisely the component whose +# own instrument is least likely to be legible. Collecting logs centrally +# makes the question answerable without host access. +# +# It is the collector that feeds this, not the services directly: one ingest +# point per swarm, same shape as the metrics path. +# +# ⚠️ NO GATEWAY VHOST, and that omission is deliberate rather than unfinished. +# VictoriaLogs' ingest and query endpoints are unauthenticated, exactly like +# the metrics store's — and the metrics store *is* published under a +# resolvable name, which is an open security question rather than a settled +# design. Publishing this one the same way would repeat that before the first +# instance is decided. The reader is Grafana, which is on this host. +{ + pkgs, + lib, + config, + ... +}: +let + cfg = config.services.hyperhive.swarm.victorialogs; + networkCfg = config.services.hyperhive.network; + hyperhiveCfg = config.services.hyperhive; +in +{ + options.services.hyperhive.swarm.victorialogs = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Run the swarm's log store on this host. Off by default and not + derived from {option}`services.hyperhive.enable`: a swarm has one + log store, so enabling it is a decision about swarm topology + rather than about whether hyperhive is installed. + ''; + }; + + package = lib.mkOption { + type = lib.types.package; + default = pkgs.victorialogs; + defaultText = lib.literalExpression "pkgs.victorialogs"; + description = "VictoriaLogs package to run."; + }; + + machine = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = "swarm-victorialogs"; + description = '' + Container name. Read-only: the name appears in host paths and in + `machinectl`, so it is a fact other modules may read rather than a + knob. + ''; + }; + + port = lib.mkOption { + type = lib.types.port; + default = 9428; + description = '' + Loopback port the store listens on. Upstream's default, kept + because there is no reason to move it and a familiar number is + one less thing an operator has to look up. + + ⚠️ Every swarm container shares the host's network namespace, so + this is a swarm-wide claim rather than a per-container one — two + modules picking the same number collide at runtime with no bind + error and nothing in any log. `state/eval-port-collisions.sh` + checks the class. + ''; + }; + + retentionPeriod = lib.mkOption { + type = lib.types.str; + default = "30d"; + example = "90d"; + description = '' + How long log data is kept. + + Deliberately far shorter than the metrics store's retention: logs + are orders of magnitude larger per unit of time, and their value + decays much faster. A log line answers "what happened during that + incident"; a metric answers "is this worse than last quarter". + ''; + }; + }; + + config = lib.mkIf (hyperhiveCfg.enable && cfg.enable) { + # This store publishes its own health as prometheus metrics on the same + # listener it serves queries on, so the swarm's collector scrapes it with + # no exporter and no extra port — same arrangement as the metrics store. + # + # Declared here rather than in the collector's module because that is the + # rule the option carries: an entry exists only where the service that + # named it runs. ⚠️ That constrains the TARGET, not the scraper — a + # deployment that splits this container away from the collector's host + # silently drops the entry, and no assertion can see it, because separate + # hosts are separate evaluations. + services.hyperhive.swarm.otel.scrapeTargets.victorialogs = "127.0.0.1:${toString cfg.port}"; + + containers.${cfg.machine} = { + autoStart = true; + ephemeral = false; + # Shared host netns, like every sibling swarm container: the collector + # and Grafana reach this at 127.0.0.1:. + privateNetwork = false; + + config = + { ... }: + { + imports = [ + (import ./swarm-container-resolver.nix { + inherit (networkCfg) bridgeIp; + dnsConsumers = [ "victorialogs.service" ]; + }) + ]; + + system.stateVersion = "26.05"; + + # This container shares the host netns, so its own firewall.service + # would rewrite the HOST ruleset at every boot. The host firewall + # owns all filtering. + networking.firewall.enable = false; + # resolvconf stays off because the resolver unit imported above + # owns /etc/resolv.conf. Leaving it on would let host-tracking + # regenerate the file empty, since the host's copy doesn't cross + # the boundary after start. + networking.resolvconf.enable = lib.mkForce false; + + services.victorialogs = { + enable = true; + package = cfg.package; + + # ⚠️ PINNED TO LOOPBACK for the same reason the metrics store is, + # and it matters more here: upstream's default listens on every + # interface, and this endpoint accepts writes as well as reads + # with no authentication of its own. The bind address is the + # boundary. + listenAddress = "127.0.0.1:${toString cfg.port}"; + + extraOptions = [ "-retentionPeriod=${cfg.retentionPeriod}" ]; + }; + }; + }; + }; + + # 🔑 OTLP ingest is served at `/insert/opentelemetry/v1/logs`, measured + # against the pinned build rather than read from docs. The path differs + # from the metrics store's `/opentelemetry/api/v1/push`, so an exporter + # configured by analogy with the metrics one is silently wrong. + # + # ⚠️ And the status code cannot tell you which is which: a wrong path, a + # right path with a wrong body, and a nonsense path ALL answer 400. Only + # the server log distinguishes them — the real route complains about the + # encoding ("json encoding isn't supported ... use protobuf"), while + # anything else logs "unsupported path requested". Verified with a + # nonsense-path control, because a probe where every arm returns the same + # code is not evidence. + # + # ⚠️ Like the metrics store's, that endpoint is unauthenticated — which is + # why `listenAddress` above is loopback, why the collector is the only + # writer, and why this module declares no gateway vhost. +}