hyperhive/nix/host-modules/swarm-victorialogs.nix

# The swarm's log store: one VictoriaLogs for the whole swarm, in a
# `swarm-victorialogs` nixos-container beside the metrics store it mirrors.
#
# Why a store at all rather than reading journals directly: an agent can
# verify that a unit was *launched* and never that it is *working*. The
# container journals are not reachable from an agent, so a diagnosis stops at
# the first component that is broken — which is precisely the component whose
# own instrument is least likely to be legible. Collecting logs centrally
# makes the question answerable without host access.
#
# It is the collector that feeds this, not the services directly: one ingest
# point per swarm, same shape as the metrics path.
#
# Gateway vhost is authenticated, unlike the metrics store's: an operator
# reaches this at `https://${cfg.domain}/` gated by the same `auth_request`
# check against authelia that swarm-ui's own vhost uses (`swarmAuthRequest`
# below, same shape as `swarm-ui.nix`'s) — see that file's copy for the full
# rationale (forceSSL is load-bearing there too: authelia answers a plain-http
# auth subrequest with 400, which `auth_request` cannot interpret as anything
# but a broken check). The store's own listener stays loopback-only and
# unauthenticated, and the vhost is now the ONLY way in from outside: the
# collector pushes through it too, at its own `= /insert/...` location, because
# a swarm has one log store and the collector need not share a host with it.
# ⚠️ That ingest location deliberately does not carry `swarmAuthRequest` — a
# pusher handed its `error_page 401 =302` follows the redirect and POSTs at a
# login page, which answers 200. See the location itself.
{
  pkgs,
  lib,
  config,
  ...
}:
let
  cfg = config.services.hyperhive.swarm.victorialogs;
  deployCfg = config.services.hyperhive.deploy;
  networkCfg = config.services.hyperhive.network;
  hyperhiveCfg = config.services.hyperhive;
  gatewayCfg = hyperhiveCfg.gateway;
  autheliaCfg = hyperhiveCfg.swarm.authelia;
  swarmDomain = hyperhiveCfg.swarm.domain;

  # Total on a null swarm domain for the same reason every sibling module is:
  # the required-domain assertion in hive-network.nix should be what an
  # operator sees, not a coercion error from here.
  domainBase = if swarmDomain == null then "invalid" else swarmDomain;

  # Copy of `swarm-ui.nix`'s own `swarmAuthRequest` — not shared code because
  # this vhost needs exactly the two locations that reference it (`/` and the
  # internal auth-request target below) and pulling in swarm-ui.nix for one
  # string would couple this module to swarm-ui existing at all, which it
  # need not. Same three lines, same reasoning as that file's own comment.
  swarmAuthRequest = ''
    auth_request /__hive_authelia;
    auth_request_set $target_url $scheme://$http_host$request_uri;
    error_page 401 =302 https://${autheliaCfg.domain}/?rd=$target_url;
  '';
in
{
  # What stays here is what the store IS from any hive's point of view: its
  # package, the name it answers on, the port. `enable` and `retentionPeriod`
  # are decisions of the host that runs it and live under `deploy.*`.
  options.services.hyperhive.swarm.victorialogs = {
    package = lib.mkOption {
      type = lib.types.package;
      default = pkgs.victorialogs;
      defaultText = lib.literalExpression "pkgs.victorialogs";
      description = "VictoriaLogs package to run.";
    };

    machine = lib.mkOption {
      type = lib.types.str;
      readOnly = true;
      default = "swarm-victorialogs";
      description = ''
        Container name. Read-only: the name appears in host paths and in
        `machinectl`, so it is a fact other modules may read rather than a
        knob.
      '';
    };

    domain = lib.mkOption {
      type = lib.types.str;
      default = "logs.${domainBase}";
      defaultText = lib.literalExpression ''"logs.''${services.hyperhive.swarm.domain}"'';
      description = ''
        Name the gateway serves this on, behind the same authelia
        `auth_request` gate as the swarm UI's own vhost. A sibling of the
        swarm's other service names, so the swarm-services sub-CA can
        issue for it — see `hive-tls.nix` for why a service name being a
        sibling rather than a child decides which CA may sign it, and
        `swarm.nix`'s `serviceDomains'` for where this name has to be
        registered for that to actually happen.
      '';
    };

    port = lib.mkOption {
      type = lib.types.port;
      default = 9428;
      description = ''
        Loopback port the store listens on. Upstream's default, kept
        because there is no reason to move it and a familiar number is
        one less thing an operator has to look up.

        ⚠️ Every swarm container shares the host's network namespace, so
        this is a swarm-wide claim rather than a per-container one — two
        modules picking the same number collide at runtime with no bind
        error and nothing in any log. `state/eval-port-collisions.sh`
        checks the class.
      '';
    };

  };

  # Retention is a property of the store this host runs, not something the
  # swarm has to agree on: it is read only where the container is defined,
  # and a hive that is a *client* of the log store never consults it. That
  # makes it a `deploy.*` value by the same rule as the seal on the secret
  # store — options on the auto-deployed service itself.
  options.services.hyperhive.deploy.victorialogs.retentionPeriod = lib.mkOption {
    type = lib.types.str;
    default = "30d";
    example = "90d";
    description = ''
      How long log data is kept.

      Deliberately far shorter than the metrics store's retention: logs
      are orders of magnitude larger per unit of time, and their value
      decays much faster. A log line answers "what happened during that
      incident"; a metric answers "is this worse than last quarter".
    '';
  };

  config = lib.mkIf (hyperhiveCfg.enable && deployCfg.victorialogs.enable) {
    # This store publishes its own health as prometheus metrics on the same
    # listener it serves queries on, so the swarm's collector scrapes it with
    # no exporter and no extra port — same arrangement as the metrics store.
    #
    # Declared here rather than in the collector's module because that is the
    # rule the option carries: an entry exists only where the service that
    # named it runs. ⚠️ That constrains the TARGET, not the scraper — a
    # deployment that splits this container away from the collector's host
    # silently drops the entry, and no assertion can see it, because separate
    # hosts are separate evaluations.
    services.hyperhive.swarm.otel.scrapeTargets.victorialogs = "127.0.0.1:${toString cfg.port}";

    # The gateway name and the quick-link, both inside `deployCfg.victorialogs.enable` — same
    # "only the host that runs the service may claim the name" guard every
    # sibling swarm-service module uses (`swarm-grafana.nix`,
    # `swarm-victoriametrics.nix`).
    services.hyperhive.gateway.localNames = [ cfg.domain ];

    # Declared here rather than in the collector, so this store's logs are
    # collected because it runs, not because a list elsewhere remembered it.
    services.hyperhive.swarm.otel.journaldUnits = [ "victorialogs" ];

    services.hyperhive.swarm.controller.links = [
      {
        label = "Logs";
        icon = "📜";
        url = "https://${cfg.domain}/";
      }
    ];

    # Authenticated front door onto the loopback-only store — see the
    # file-top comment for why this is safe to add without touching the
    # store's own (still unauthenticated, still loopback) listener at all.
    # `removeAttrs`/`forceSSL`: same asymmetry `swarm-ui.nix` documents —
    # authelia answers a plain-http auth subrequest with 400, which
    # `auth_request` cannot read as anything but a broken check, so this
    # vhost needs `forceSSL` rather than the `addSSL` every unauthenticated
    # sibling vhost uses.
    services.nginx.virtualHosts."${cfg.domain}" =
      (builtins.removeAttrs (gatewayCfg.lib.tlsFor cfg.domain) [ "addSSL" ])
      // {
        forceSSL = true;
        listen = gatewayCfg.lib.listen;
        extraConfig = gatewayCfg.lib.securityHeaders;
        locations = {
          "/" = {
            proxyPass = "http://127.0.0.1:${toString cfg.port}/";
            extraConfig = swarmAuthRequest;
          };

          # The swarm collector's ingest route. `=` so it outranks the `/`
          # prefix above — without a more specific location it would ride that
          # catch-all, and that is the whole hazard here.
          #
          # ⚠️ Deliberately NOT `swarmAuthRequest`. That block ends in
          # `error_page 401 =302`, which is right for a browser and wrong for a
          # pusher: handed a redirect it follows the redirect and POSTs its
          # batch at a login page, which answers 200. Ingest then reports
          # healthy while storing nothing. A machine route lets the 401 reach
          # the client unchanged.
          #
          # `auth_request` does not inherit across sibling locations (see the
          # note on `swarmAuthRequest` above), so naming it here is required
          # rather than redundant.
          "= /insert/opentelemetry/v1/logs" = {
            proxyPass = "http://127.0.0.1:${toString cfg.port}/insert/opentelemetry/v1/logs";
            extraConfig = ''
              auth_request /__hive_authelia;
            '';
          };
          # The subrequest itself — same target, same header set, same
          # reasoning as `swarm-ui.nix`'s own copy (measured against the
          # pinned authelia binary, not copied from an example).
          "= /__hive_authelia" = {
            proxyPass = "https://${autheliaCfg.domain}/api/authz/auth-request";
            # nixpkgs appends its OWN `Host $host` after extraConfig,
            # which would override verifiedProxyTo's — see the comment
            # on verifiedProxyTo in hive-gateway/vhost-lib.nix.
            recommendedProxySettings = false;
            extraConfig = ''
              internal;
              ${gatewayCfg.lib.verifiedProxyTo autheliaCfg.domain}
              proxy_pass_request_body off;
              proxy_set_header Content-Length "";
              proxy_set_header X-Original-Method $request_method;
              proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
              proxy_set_header X-Forwarded-Proto $scheme;
              proxy_set_header X-Forwarded-Host $http_host;
              proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            '';
          };
        };
      };

    containers.${cfg.machine} = {
      autoStart = true;
      ephemeral = false;
      # Journal files on the host, not inside the container: nixpkgs hardcodes
      # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
      extraFlags = [ "--link-journal=host" ];
      # Shared host netns, like every sibling swarm container: the collector
      # and Grafana reach this at 127.0.0.1:<port>.
      privateNetwork = false;

      config =
        { ... }:
        {
          imports = [
            (import ./swarm-container-resolver.nix {
              inherit (networkCfg) bridgeIp;
              dnsConsumers = [ "victorialogs.service" ];
            })
          ];

          system.stateVersion = "26.05";

          # This container shares the host netns, so its own firewall.service
          # would rewrite the HOST ruleset at every boot. The host firewall
          # owns all filtering.
          networking.firewall.enable = false;
          # resolvconf stays off because the resolver unit imported above
          # owns /etc/resolv.conf. Leaving it on would let host-tracking
          # regenerate the file empty, since the host's copy doesn't cross
          # the boundary after start.
          networking.resolvconf.enable = lib.mkForce false;

          services.victorialogs = {
            enable = true;
            package = cfg.package;

            # ⚠️ PINNED TO LOOPBACK for the same reason the metrics store is,
            # and it matters more here: upstream's default listens on every
            # interface, and this endpoint accepts writes as well as reads
            # with no authentication of its own. The bind address is the
            # boundary.
            listenAddress = "127.0.0.1:${toString cfg.port}";

            extraOptions = [ "-retentionPeriod=${deployCfg.victorialogs.retentionPeriod}" ];
          };
        };
    };
  };

  # 🔑 OTLP ingest is served at `/insert/opentelemetry/v1/logs`, measured
  # against the pinned build rather than read from docs. The path differs
  # from the metrics store's `/opentelemetry/api/v1/push`, so an exporter
  # configured by analogy with the metrics one is silently wrong.
  #
  # ⚠️ And the status code cannot tell you which is which: a wrong path, a
  # right path with a wrong body, and a nonsense path ALL answer 400. Only
  # the server log distinguishes them — the real route complains about the
  # encoding ("json encoding isn't supported ... use protobuf"), while
  # anything else logs "unsupported path requested". Verified with a
  # nonsense-path control, because a probe where every arm returns the same
  # code is not evidence.
  #
  # ⚠️ Like the metrics store's, that endpoint is unauthenticated — which is
  # why `listenAddress` above is loopback and why nothing reaches it except
  # through the gateway, where the ingest route is authenticated. (This module
  # does declare a vhost; the line that used to say otherwise was already
  # wrong before the collector started pushing through it.)
}