# The swarm's log store: one VictoriaLogs for the whole swarm, in a # `swarm-victorialogs` nixos-container beside the metrics store it mirrors. # # Why a store at all rather than reading journals directly: an agent can # verify that a unit was *launched* and never that it is *working*. The # container journals are not reachable from an agent, so a diagnosis stops at # the first component that is broken — which is precisely the component whose # own instrument is least likely to be legible. Collecting logs centrally # makes the question answerable without host access. # # It is the collector that feeds this, not the services directly: one ingest # point per swarm, same shape as the metrics path. # # ⚠️ NO GATEWAY VHOST, and that omission is deliberate rather than unfinished. # VictoriaLogs' ingest and query endpoints are unauthenticated, exactly like # the metrics store's — and the metrics store *is* published under a # resolvable name, which is an open security question rather than a settled # design. Publishing this one the same way would repeat that before the first # instance is decided. The reader is Grafana, which is on this host. { pkgs, lib, config, ... }: let cfg = config.services.hyperhive.swarm.victorialogs; networkCfg = config.services.hyperhive.network; hyperhiveCfg = config.services.hyperhive; in { options.services.hyperhive.swarm.victorialogs = { enable = lib.mkOption { type = lib.types.bool; default = false; description = '' Run the swarm's log store on this host. Off by default and not derived from {option}`services.hyperhive.enable`: a swarm has one log store, so enabling it is a decision about swarm topology rather than about whether hyperhive is installed. ''; }; package = lib.mkOption { type = lib.types.package; default = pkgs.victorialogs; defaultText = lib.literalExpression "pkgs.victorialogs"; description = "VictoriaLogs package to run."; }; machine = lib.mkOption { type = lib.types.str; readOnly = true; default = "swarm-victorialogs"; description = '' Container name. Read-only: the name appears in host paths and in `machinectl`, so it is a fact other modules may read rather than a knob. ''; }; port = lib.mkOption { type = lib.types.port; default = 9428; description = '' Loopback port the store listens on. Upstream's default, kept because there is no reason to move it and a familiar number is one less thing an operator has to look up. ⚠️ Every swarm container shares the host's network namespace, so this is a swarm-wide claim rather than a per-container one — two modules picking the same number collide at runtime with no bind error and nothing in any log. `state/eval-port-collisions.sh` checks the class. ''; }; retentionPeriod = lib.mkOption { type = lib.types.str; default = "30d"; example = "90d"; description = '' How long log data is kept. Deliberately far shorter than the metrics store's retention: logs are orders of magnitude larger per unit of time, and their value decays much faster. A log line answers "what happened during that incident"; a metric answers "is this worse than last quarter". ''; }; }; config = lib.mkIf (hyperhiveCfg.enable && cfg.enable) { # This store publishes its own health as prometheus metrics on the same # listener it serves queries on, so the swarm's collector scrapes it with # no exporter and no extra port — same arrangement as the metrics store. # # Declared here rather than in the collector's module because that is the # rule the option carries: an entry exists only where the service that # named it runs. ⚠️ That constrains the TARGET, not the scraper — a # deployment that splits this container away from the collector's host # silently drops the entry, and no assertion can see it, because separate # hosts are separate evaluations. services.hyperhive.swarm.otel.scrapeTargets.victorialogs = "127.0.0.1:${toString cfg.port}"; containers.${cfg.machine} = { autoStart = true; ephemeral = false; # Shared host netns, like every sibling swarm container: the collector # and Grafana reach this at 127.0.0.1:. privateNetwork = false; config = { ... }: { imports = [ (import ./swarm-container-resolver.nix { inherit (networkCfg) bridgeIp; dnsConsumers = [ "victorialogs.service" ]; }) ]; system.stateVersion = "26.05"; # This container shares the host netns, so its own firewall.service # would rewrite the HOST ruleset at every boot. The host firewall # owns all filtering. networking.firewall.enable = false; # resolvconf stays off because the resolver unit imported above # owns /etc/resolv.conf. Leaving it on would let host-tracking # regenerate the file empty, since the host's copy doesn't cross # the boundary after start. networking.resolvconf.enable = lib.mkForce false; services.victorialogs = { enable = true; package = cfg.package; # ⚠️ PINNED TO LOOPBACK for the same reason the metrics store is, # and it matters more here: upstream's default listens on every # interface, and this endpoint accepts writes as well as reads # with no authentication of its own. The bind address is the # boundary. listenAddress = "127.0.0.1:${toString cfg.port}"; extraOptions = [ "-retentionPeriod=${cfg.retentionPeriod}" ]; }; }; }; }; # 🔑 OTLP ingest is served at `/insert/opentelemetry/v1/logs`, measured # against the pinned build rather than read from docs. The path differs # from the metrics store's `/opentelemetry/api/v1/push`, so an exporter # configured by analogy with the metrics one is silently wrong. # # ⚠️ And the status code cannot tell you which is which: a wrong path, a # right path with a wrong body, and a nonsense path ALL answer 400. Only # the server log distinguishes them — the real route complains about the # encoding ("json encoding isn't supported ... use protobuf"), while # anything else logs "unsupported path requested". Verified with a # nonsense-path control, because a probe where every arm returns the same # code is not evidence. # # ⚠️ Like the metrics store's, that endpoint is unauthenticated — which is # why `listenAddress` above is loopback, why the collector is the only # writer, and why this module declares no gateway vhost. }