hyperhive/nix/host-modules/swarm-victoriametrics.nix

# The swarm's metrics store: one VictoriaMetrics for the whole swarm, in a
# `swarm-victoriametrics` nixos-container.
#
# A LOCAL time-series database rather than only an external sink, and that is
# the point rather than a convenience: the swarm dashboard has to stay
# readable when the outside world is unreachable. Same failure-domain property
# the hive-status KV has — a view of the system must not depend on the system
# it is viewing being healthy.
#
# It is the collector that feeds this (the gateway OTEL collector), not the
# agents directly: one ingest point per swarm, authenticated there.
{
  pkgs,
  lib,
  config,
  ...
}:
let
  cfg = config.services.hyperhive.swarm.victoriametrics;
  deployCfg = config.services.hyperhive.deploy;
  hyperhiveCfg = config.services.hyperhive;
  gatewayCfg = hyperhiveCfg.gateway;

  # A netns of its own with no interface but loopback: the store makes no
  # outbound calls, and the gateway reaches it over `socketPath` below, so
  # nothing on the host or the hive bridge can dial its listener.
  privateNetwork = true;

  # Fixed, not an option: the directory's mode is what keeps the socket
  # private, so it holds this one socket and nothing else.
  socketDir = "/run/swarm-victoriametrics";
  socketPath = "${socketDir}/victoriametrics.sock";
  upstream = path: "http://unix:${socketPath}:${path}";

  # The gid the gateway's nginx runs as. Only the number crosses into the
  # container, which knows it as `gateway-nginx`.
  nginxGid = config.ids.gids.nginx;

  # The store's read API: the only part of it the vhost serves without a
  # credential. Each location's `proxy_pass` carries the path, so the store
  # receives the path nginx matched, not the caller's unnormalised spelling.
  #
  # Grafana's Prometheus datasource reads the query, series, label, metadata,
  # buildinfo, exemplar, rules and alerts routes; the store answers the last
  # three with fixed placeholders. vmui is the static files under `/vmui/`
  # plus the query API and the remaining read-only routes here. Paths read off
  # the pinned build's `app/vmselect/main.go` and vmui's `src/`.
  readPaths = [
    "/api/v1/query"
    "/api/v1/query_range"
    "/api/v1/query_exemplars"
    "/api/v1/series"
    "/api/v1/labels"
    "/api/v1/metadata"
    "/api/v1/rules"
    "/api/v1/alerts"
    "/api/v1/status/buildinfo"
    "/api/v1/export"
    "/api/v1/export/csv"
    "/api/v1/status/tsdb"
    "/api/v1/status/metric_names_stats"
    "/api/v1/status/top_queries"
    "/api/v1/status/active_queries"
    "/expand-with-exprs"
    "/prettify-query"
    "/metric-relabel-debug"
    "/vmui"
  ];
  # The store serves only `<name>/values` under `/api/v1/label/`, and only
  # static files and UI config under `/vmui/`.
  readPrefixes = [
    "/api/v1/label/"
    "/vmui/"
  ];
  readLocations =
    let
      to = path: { proxyPass = upstream path; };
    in
    lib.listToAttrs (
      map (p: lib.nameValuePair "= ${p}" (to p)) readPaths
      ++ map (p: lib.nameValuePair "^~ ${p}" (to p)) readPrefixes
    );
in
{
  # What the store IS from any hive's point of view, the name it answers on,
  # is `swarm.victoriametrics` in ./swarm-victoriametrics-service.nix.
  # `enable`, `package` and `retentionPeriod` are decisions of the host that
  # runs it and live under `deploy.*`.

  # Retention is a property of the store this host runs, not something the
  # swarm has to agree on: it is read only where the container is defined,
  # and a hive that is a *client* of the metrics store never consults it.
  options.services.hyperhive.deploy.victoriametrics.package = lib.mkOption {
    type = lib.types.package;
    default = pkgs.victoriametrics;
    defaultText = lib.literalExpression "pkgs.victoriametrics";
    description = "VictoriaMetrics package to run.";
  };

  options.services.hyperhive.deploy.victoriametrics.retentionPeriod = lib.mkOption {
    type = lib.types.str;
    default = "5y";
    example = "90d";
    description = ''
      How long samples are kept.

      Deliberately a high default rather than a required option: the two
      failure directions are not symmetric. Too long fills a disk, which
      is visible and recoverable by lowering this; too short **destroys
      history**, silently and permanently. So the safe default is generous
      and an operator lowers it once they have measured how fast this swarm
      actually accumulates data.
    '';
  };

  config = lib.mkIf deployCfg.victoriametrics.enable {
    # The gateway name, inside `deployCfg.victoriametrics.enable` — that
    # guard is the load-bearing part. Every hive in a swarm may know this
    # store exists, but only the host that RUNS it may claim the name; a
    # client hive declaring the vhost would answer for a service it does not
    # have.
    services.hyperhive.gateway.localNames = [ cfg.domain ];

    # This host serves the vhost, and the hive's dnsmasq answers its name.
    services.hyperhive.gateway.enable = lib.mkDefault true;
    services.hyperhive.gateway.dns.enable = lib.mkDefault true;

    services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // {
      listen = gatewayCfg.lib.listen;
      extraConfig = gatewayCfg.lib.securityHeaders;
      locations = readLocations // {
        # Nothing reaches the store unless a location here names it. The
        # store serves writes and admin calls (`/api/v1/import`,
        # `/api/v1/admin/tsdb/delete_series`, ...) on the same listener as
        # its queries, and authenticates none of them.
        "/".return = "403";

        # The swarm UI's quick link opens the bare domain. The store's own
        # index page links mostly to paths refused above, so send it to vmui.
        "= /".return = "302 /vmui/";

        # The swarm collector's ingest route. `=` so it outranks the `/`
        # prefix above.
        #
        # ⚠️ No `error_page 401 =302` here, and its absence is the point: a
        # redirect is right for a browser and wrong for a pusher, which would
        # follow it and POST its batch at a login page that answers 200 —
        # ingest reporting healthy while storing nothing.
        "= /opentelemetry/api/v1/push" = {
          proxyPass = upstream "/opentelemetry/api/v1/push";
          extraConfig = ''
            auth_request /__metrics_push_authz;
          '';
        };

        # The store's own health, for the swarm collector wherever it runs:
        # served on the query listener, so no exporter and no extra port. `=`
        # for the same reason as the push route, and no `error_page` for the
        # same reason: a scraper would parse a login page as metrics.
        #
        # The collector requests exactly this URL, declared as
        # `publishedScrapeTargets.victoriametrics` in ./swarm-otel-service.nix;
        # the two must agree, since that string is also the token's audience.
        "= /metrics" = {
          proxyPass = upstream "/metrics";
          extraConfig = ''
            auth_request /__metrics_push_authz;
          '';
        };

        # The subrequest for both gated locations above. Authelia admits the
        # collector's client anywhere on this name and refuses every other
        # subject (`metricsStoreRules` in ./swarm-authelia.nix), so the scrape
        # needs no subrequest of its own. Same target and header set as the
        # sibling log store's, which took them from `swarm-ui.nix` —
        # `X-Original-URL` and `X-Original-Method` are what authelia's
        # auth-request implementation reads, and the address it compares the
        # token's audience against.
        "= /__metrics_push_authz" = {
          proxyPass = "https://${hyperhiveCfg.swarm.authelia.domain}/api/authz/auth-request";
          # nixpkgs appends its OWN `Host $host` after extraConfig, which
          # would override verifiedProxyTo's — see the comment on
          # verifiedProxyTo in hive-gateway/vhost-lib.nix.
          recommendedProxySettings = false;
          extraConfig = ''
            internal;
            ${gatewayCfg.lib.verifiedProxyTo hyperhiveCfg.swarm.authelia.domain}
            proxy_pass_request_body off;
            proxy_set_header Content-Length "";
            proxy_set_header X-Original-Method $request_method;
            proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_set_header X-Forwarded-Host $http_host;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
          '';
        };
      };
    };

    # The socket directory, created host-side before the container starts:
    # nixos-container refuses to start when a bind source is missing.
    #
    # Root-owned and group nginx, `0770`: the store creates its socket here
    # through the group (its uid is allocated inside the container, so the
    # host cannot name it), and the gateway traverses it the same way.
    # Everything else is kept out.
    systemd.tmpfiles.rules = [
      "d ${socketDir} 0770 root ${toString nginxGid} - -"
    ];

    containers.${cfg.machine} = {
      autoStart = true;
      ephemeral = false;
      # Journal files on the host, not inside the container: nixpkgs hardcodes
      # --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
      extraFlags = [ "--link-journal=host" ];
      inherit privateNetwork;

      # A bind mount rather than a path both sides happen to know: a
      # container's `/run` is an nspawn tmpfs the host cannot see into.
      bindMounts.${socketDir} = {
        hostPath = socketDir;
        isReadOnly = false;
      };

      config =
        { pkgs, ... }:
        {
          imports = [ ../container-modules/swarm-container.nix ];

          # No resolver unit: with no interface but loopback there is no
          # nameserver to reach, and the store resolves no names.
          services.hyperhive.swarmContainer = {
            inherit privateNetwork;
            writesOwnResolvConf = false;
          };

          # The socket's group is the store's primary group, so that group is
          # nginx's gid. A static user, not nixpkgs' DynamicUser: a dynamic
          # user's group is allocated per start, and nginx is never in it.
          users.groups.gateway-nginx.gid = nginxGid;
          users.users.victoriametrics = {
            isSystemUser = true;
            group = "gateway-nginx";
          };

          systemd.services.victoriametrics = {
            serviceConfig = {
              DynamicUser = lib.mkForce false;
              User = "victoriametrics";
              Group = "gateway-nginx";
              # The store creates its socket under the process umask, so
              # `0007` makes it `0770` and nginx's group can connect.
              UMask = "0007";
            };
            # nixpkgs' readiness loop curls `http://<listenAddress>/ping`,
            # which no `unix:` address answers.
            postStart = lib.mkForce ''
              until ${lib.getExe pkgs.curl} -s -o /dev/null --unix-socket ${socketPath} http://localhost/ping; do
                sleep 1;
              done
            '';
          };

          services.victoriametrics = {
            enable = true;
            package = deployCfg.victoriametrics.package;
            retentionPeriod = deployCfg.victoriametrics.retentionPeriod;

            # ⚠️ A socket, not upstream's default `:8428` on every interface:
            # the gateway is the only intended client, and this listener
            # carries an unauthenticated write endpoint (see the OTLP note
            # below).
            listenAddress = "unix:${socketPath}";
          };
        };
    };
  };

  # 🔑 OTLP ingest needs no flag. Measured against the pinned 1.146.0 rather
  # than inferred from the module's option list, which has no OTLP switch and
  # so reads as though the feature were absent: the running server answers
  # `POST /opentelemetry/api/v1/push` with 200 (a nonexistent path answers
  # 400, so that 200 means the route exists). The `-opentelemetry.*` flags
  # only tune naming and limits, and are reachable via `extraOptions` if a
  # deployment ever needs them.
  #
  # ⚠️ That endpoint is unauthenticated, which is why `listenAddress` above is
  # a socket only the gateway can reach: the only route to it is the vhost's
  # ingest location, which authenticates. The collector is still the only writer, but
  # it now arrives by the swarm name rather than over loopback, because it need
  # not share a host with this store.
}