diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index 666490b5..da0225c9 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -30,6 +30,7 @@ ./swarm-otel.nix ./swarm-snapshot-store.nix ./swarm-ui.nix + ./swarm-victorialogs.nix ./swarm-victoriametrics.nix ./swarm-wireguard.nix ./swarm.nix diff --git a/nix/host-modules/swarm-victorialogs.nix b/nix/host-modules/swarm-victorialogs.nix new file mode 100644 index 00000000..3e1e4781 --- /dev/null +++ b/nix/host-modules/swarm-victorialogs.nix @@ -0,0 +1,168 @@ +# The swarm's log store: one VictoriaLogs for the whole swarm, in a +# `swarm-victorialogs` nixos-container beside the metrics store it mirrors. +# +# Why a store at all rather than reading journals directly: an agent can +# verify that a unit was *launched* and never that it is *working*. The +# container journals are not reachable from an agent, so a diagnosis stops at +# the first component that is broken — which is precisely the component whose +# own instrument is least likely to be legible. Collecting logs centrally +# makes the question answerable without host access. +# +# It is the collector that feeds this, not the services directly: one ingest +# point per swarm, same shape as the metrics path. +# +# ⚠️ NO GATEWAY VHOST, and that omission is deliberate rather than unfinished. +# VictoriaLogs' ingest and query endpoints are unauthenticated, exactly like +# the metrics store's — and the metrics store *is* published under a +# resolvable name, which is an open security question rather than a settled +# design. Publishing this one the same way would repeat that before the first +# instance is decided. The reader is Grafana, which is on this host. +{ + pkgs, + lib, + config, + ... +}: +let + cfg = config.services.hyperhive.swarm.victorialogs; + networkCfg = config.services.hyperhive.network; + hyperhiveCfg = config.services.hyperhive; +in +{ + options.services.hyperhive.swarm.victorialogs = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Run the swarm's log store on this host. Off by default and not + derived from {option}`services.hyperhive.enable`: a swarm has one + log store, so enabling it is a decision about swarm topology + rather than about whether hyperhive is installed. + ''; + }; + + package = lib.mkOption { + type = lib.types.package; + default = pkgs.victorialogs; + defaultText = lib.literalExpression "pkgs.victorialogs"; + description = "VictoriaLogs package to run."; + }; + + machine = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = "swarm-victorialogs"; + description = '' + Container name. Read-only: the name appears in host paths and in + `machinectl`, so it is a fact other modules may read rather than a + knob. + ''; + }; + + port = lib.mkOption { + type = lib.types.port; + default = 9428; + description = '' + Loopback port the store listens on. Upstream's default, kept + because there is no reason to move it and a familiar number is + one less thing an operator has to look up. + + ⚠️ Every swarm container shares the host's network namespace, so + this is a swarm-wide claim rather than a per-container one — two + modules picking the same number collide at runtime with no bind + error and nothing in any log. `state/eval-port-collisions.sh` + checks the class. + ''; + }; + + retentionPeriod = lib.mkOption { + type = lib.types.str; + default = "30d"; + example = "90d"; + description = '' + How long log data is kept. + + Deliberately far shorter than the metrics store's retention: logs + are orders of magnitude larger per unit of time, and their value + decays much faster. A log line answers "what happened during that + incident"; a metric answers "is this worse than last quarter". + ''; + }; + }; + + config = lib.mkIf (hyperhiveCfg.enable && cfg.enable) { + # This store publishes its own health as prometheus metrics on the same + # listener it serves queries on, so the swarm's collector scrapes it with + # no exporter and no extra port — same arrangement as the metrics store. + # + # Declared here rather than in the collector's module because that is the + # rule the option carries: an entry exists only where the service that + # named it runs. ⚠️ That constrains the TARGET, not the scraper — a + # deployment that splits this container away from the collector's host + # silently drops the entry, and no assertion can see it, because separate + # hosts are separate evaluations. + services.hyperhive.swarm.otel.scrapeTargets.victorialogs = "127.0.0.1:${toString cfg.port}"; + + containers.${cfg.machine} = { + autoStart = true; + ephemeral = false; + # Shared host netns, like every sibling swarm container: the collector + # and Grafana reach this at 127.0.0.1:. + privateNetwork = false; + + config = + { ... }: + { + imports = [ + (import ./swarm-container-resolver.nix { + inherit (networkCfg) bridgeIp; + dnsConsumers = [ "victorialogs.service" ]; + }) + ]; + + system.stateVersion = "26.05"; + + # This container shares the host netns, so its own firewall.service + # would rewrite the HOST ruleset at every boot. The host firewall + # owns all filtering. + networking.firewall.enable = false; + # resolvconf stays off because the resolver unit imported above + # owns /etc/resolv.conf. Leaving it on would let host-tracking + # regenerate the file empty, since the host's copy doesn't cross + # the boundary after start. + networking.resolvconf.enable = lib.mkForce false; + + services.victorialogs = { + enable = true; + package = cfg.package; + + # ⚠️ PINNED TO LOOPBACK for the same reason the metrics store is, + # and it matters more here: upstream's default listens on every + # interface, and this endpoint accepts writes as well as reads + # with no authentication of its own. The bind address is the + # boundary. + listenAddress = "127.0.0.1:${toString cfg.port}"; + + extraOptions = [ "-retentionPeriod=${cfg.retentionPeriod}" ]; + }; + }; + }; + }; + + # 🔑 OTLP ingest is served at `/insert/opentelemetry/v1/logs`, measured + # against the pinned build rather than read from docs. The path differs + # from the metrics store's `/opentelemetry/api/v1/push`, so an exporter + # configured by analogy with the metrics one is silently wrong. + # + # ⚠️ And the status code cannot tell you which is which: a wrong path, a + # right path with a wrong body, and a nonsense path ALL answer 400. Only + # the server log distinguishes them — the real route complains about the + # encoding ("json encoding isn't supported ... use protobuf"), while + # anything else logs "unsupported path requested". Verified with a + # nonsense-path control, because a probe where every arm returns the same + # code is not evidence. + # + # ⚠️ Like the metrics store's, that endpoint is unauthenticated — which is + # why `listenAddress` above is loopback, why the collector is the only + # writer, and why this module declares no gateway vhost. +}