# The swarm's log store: one VictoriaLogs for the whole swarm, in a # `swarm-victorialogs` nixos-container beside the metrics store it mirrors. # # Why a store at all rather than reading journals directly: an agent can # verify that a unit was *launched* and never that it is *working*. The # container journals are not reachable from an agent, so a diagnosis stops at # the first component that is broken — which is precisely the component whose # own instrument is least likely to be legible. Collecting logs centrally # makes the question answerable without host access. # # It is the collector that feeds this, not the services directly: one ingest # point per swarm, same shape as the metrics path. # # Gateway vhost is authenticated, unlike the metrics store's: an operator # reaches this at `https://${cfg.domain}/` gated by the same `auth_request` # check against authelia that swarm-ui's own vhost uses (`swarmAuthRequest` # below, same shape as `swarm-ui.nix`'s) — see that file's copy for the full # rationale (forceSSL is load-bearing there too: authelia answers a plain-http # auth subrequest with 400, which `auth_request` cannot interpret as anything # but a broken check). The store's own listener stays loopback-only and # unauthenticated exactly as before — the collector still writes to it # directly, never through this vhost — so this adds a new authenticated front # door without touching the existing write path at all. { pkgs, lib, config, ... }: let cfg = config.services.hyperhive.swarm.victorialogs; deployCfg = config.services.hyperhive.deploy; networkCfg = config.services.hyperhive.network; hyperhiveCfg = config.services.hyperhive; gatewayCfg = hyperhiveCfg.gateway; autheliaCfg = hyperhiveCfg.swarm.authelia; swarmDomain = hyperhiveCfg.swarm.domain; # Total on a null swarm domain for the same reason every sibling module is: # the required-domain assertion in hive-network.nix should be what an # operator sees, not a coercion error from here. domainBase = if swarmDomain == null then "invalid" else swarmDomain; # Copy of `swarm-ui.nix`'s own `swarmAuthRequest` — not shared code because # this vhost needs exactly the two locations that reference it (`/` and the # internal auth-request target below) and pulling in swarm-ui.nix for one # string would couple this module to swarm-ui existing at all, which it # need not. Same three lines, same reasoning as that file's own comment. swarmAuthRequest = '' auth_request /__hive_authelia; auth_request_set $target_url $scheme://$http_host$request_uri; error_page 401 =302 https://${autheliaCfg.domain}/?rd=$target_url; ''; in { # What stays here is what the store IS from any hive's point of view: its # package, the name it answers on, the port. `enable` and `retentionPeriod` # are decisions of the host that runs it and live under `deploy.*`. options.services.hyperhive.swarm.victorialogs = { package = lib.mkOption { type = lib.types.package; default = pkgs.victorialogs; defaultText = lib.literalExpression "pkgs.victorialogs"; description = "VictoriaLogs package to run."; }; machine = lib.mkOption { type = lib.types.str; readOnly = true; default = "swarm-victorialogs"; description = '' Container name. Read-only: the name appears in host paths and in `machinectl`, so it is a fact other modules may read rather than a knob. ''; }; domain = lib.mkOption { type = lib.types.str; default = "logs.${domainBase}"; defaultText = lib.literalExpression ''"logs.''${services.hyperhive.swarm.domain}"''; description = '' Name the gateway serves this on, behind the same authelia `auth_request` gate as the swarm UI's own vhost. A sibling of the swarm's other service names, so the swarm-services sub-CA can issue for it — see `hive-tls.nix` for why a service name being a sibling rather than a child decides which CA may sign it, and `swarm.nix`'s `serviceDomains'` for where this name has to be registered for that to actually happen. ''; }; port = lib.mkOption { type = lib.types.port; default = 9428; description = '' Loopback port the store listens on. Upstream's default, kept because there is no reason to move it and a familiar number is one less thing an operator has to look up. ⚠️ Every swarm container shares the host's network namespace, so this is a swarm-wide claim rather than a per-container one — two modules picking the same number collide at runtime with no bind error and nothing in any log. `state/eval-port-collisions.sh` checks the class. ''; }; }; # Retention is a property of the store this host runs, not something the # swarm has to agree on: it is read only where the container is defined, # and a hive that is a *client* of the log store never consults it. That # makes it a `deploy.*` value by the same rule as the seal on the secret # store — options on the auto-deployed service itself. options.services.hyperhive.deploy.victorialogs.retentionPeriod = lib.mkOption { type = lib.types.str; default = "30d"; example = "90d"; description = '' How long log data is kept. Deliberately far shorter than the metrics store's retention: logs are orders of magnitude larger per unit of time, and their value decays much faster. A log line answers "what happened during that incident"; a metric answers "is this worse than last quarter". ''; }; config = lib.mkIf (hyperhiveCfg.enable && deployCfg.victorialogs.enable) { # This store publishes its own health as prometheus metrics on the same # listener it serves queries on, so the swarm's collector scrapes it with # no exporter and no extra port — same arrangement as the metrics store. # # Declared here rather than in the collector's module because that is the # rule the option carries: an entry exists only where the service that # named it runs. ⚠️ That constrains the TARGET, not the scraper — a # deployment that splits this container away from the collector's host # silently drops the entry, and no assertion can see it, because separate # hosts are separate evaluations. services.hyperhive.swarm.otel.scrapeTargets.victorialogs = "127.0.0.1:${toString cfg.port}"; # The gateway name and the quick-link, both inside `deployCfg.victorialogs.enable` — same # "only the host that runs the service may claim the name" guard every # sibling swarm-service module uses (`swarm-grafana.nix`, # `swarm-victoriametrics.nix`). services.hyperhive.gateway.localNames = [ cfg.domain ]; # Declared here rather than in the collector, so this store's logs are # collected because it runs, not because a list elsewhere remembered it. services.hyperhive.swarm.otel.journaldUnits = [ "victorialogs" ]; services.hyperhive.swarm.controller.links = [ { label = "Logs"; icon = "📜"; url = "https://${cfg.domain}/"; } ]; # Authenticated front door onto the loopback-only store — see the # file-top comment for why this is safe to add without touching the # store's own (still unauthenticated, still loopback) listener at all. # `removeAttrs`/`forceSSL`: same asymmetry `swarm-ui.nix` documents — # authelia answers a plain-http auth subrequest with 400, which # `auth_request` cannot read as anything but a broken check, so this # vhost needs `forceSSL` rather than the `addSSL` every unauthenticated # sibling vhost uses. services.nginx.virtualHosts."${cfg.domain}" = (builtins.removeAttrs (gatewayCfg.lib.tlsFor cfg.domain) [ "addSSL" ]) // { forceSSL = true; listen = gatewayCfg.lib.listen; extraConfig = gatewayCfg.lib.securityHeaders; locations = { "/" = { proxyPass = "http://127.0.0.1:${toString cfg.port}/"; extraConfig = swarmAuthRequest; }; # The subrequest itself — same target, same header set, same # reasoning as `swarm-ui.nix`'s own copy (measured against the # pinned authelia binary, not copied from an example). "= /__hive_authelia" = { proxyPass = "https://${autheliaCfg.domain}/api/authz/auth-request"; # nixpkgs appends its OWN `Host $host` after extraConfig, # which would override verifiedProxyTo's — see the comment # on verifiedProxyTo in hive-gateway/vhost-lib.nix. recommendedProxySettings = false; extraConfig = '' internal; ${gatewayCfg.lib.verifiedProxyTo autheliaCfg.domain} proxy_pass_request_body off; proxy_set_header Content-Length ""; proxy_set_header X-Original-Method $request_method; proxy_set_header X-Original-URL $scheme://$http_host$request_uri; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Host $http_host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; ''; }; }; }; containers.${cfg.machine} = { autoStart = true; ephemeral = false; # Shared host netns, like every sibling swarm container: the collector # and Grafana reach this at 127.0.0.1:. privateNetwork = false; config = { ... }: { imports = [ (import ./swarm-container-resolver.nix { inherit (networkCfg) bridgeIp; dnsConsumers = [ "victorialogs.service" ]; }) ]; system.stateVersion = "26.05"; # This container shares the host netns, so its own firewall.service # would rewrite the HOST ruleset at every boot. The host firewall # owns all filtering. networking.firewall.enable = false; # resolvconf stays off because the resolver unit imported above # owns /etc/resolv.conf. Leaving it on would let host-tracking # regenerate the file empty, since the host's copy doesn't cross # the boundary after start. networking.resolvconf.enable = lib.mkForce false; services.victorialogs = { enable = true; package = cfg.package; # ⚠️ PINNED TO LOOPBACK for the same reason the metrics store is, # and it matters more here: upstream's default listens on every # interface, and this endpoint accepts writes as well as reads # with no authentication of its own. The bind address is the # boundary. listenAddress = "127.0.0.1:${toString cfg.port}"; extraOptions = [ "-retentionPeriod=${deployCfg.victorialogs.retentionPeriod}" ]; }; }; }; }; # 🔑 OTLP ingest is served at `/insert/opentelemetry/v1/logs`, measured # against the pinned build rather than read from docs. The path differs # from the metrics store's `/opentelemetry/api/v1/push`, so an exporter # configured by analogy with the metrics one is silently wrong. # # ⚠️ And the status code cannot tell you which is which: a wrong path, a # right path with a wrong body, and a nonsense path ALL answer 400. Only # the server log distinguishes them — the real route complains about the # encoding ("json encoding isn't supported ... use protobuf"), while # anything else logs "unsupported path requested". Verified with a # nonsense-path control, because a probe where every arm returns the same # code is not evidence. # # ⚠️ Like the metrics store's, that endpoint is unauthenticated — which is # why `listenAddress` above is loopback, why the collector is the only # writer, and why this module declares no gateway vhost. }