4f4f1ec3ee
Both retentionPeriods move to deploy.<store>.retentionPeriod. Measured before moving: each is read exactly once, in the container definition of the module that deploys the store, and no client hive consults either -- so they are options on the auto-deployed service rather than something every host has to agree on. What stays in swarm.* is what a client needs to reach the store: the package, the name it answers on, the port. Both modules said "what stays here is its package, domain, retention and wiring" in their options-block comments; retention no longer does, so those say so. Renames go in deploy.nix with the rest, so the whole move keeps one home and one file to delete when the deprecation window closes.
266 lines
12 KiB
Nix
266 lines
12 KiB
Nix
# The swarm's log store: one VictoriaLogs for the whole swarm, in a
|
||
# `swarm-victorialogs` nixos-container beside the metrics store it mirrors.
|
||
#
|
||
# Why a store at all rather than reading journals directly: an agent can
|
||
# verify that a unit was *launched* and never that it is *working*. The
|
||
# container journals are not reachable from an agent, so a diagnosis stops at
|
||
# the first component that is broken — which is precisely the component whose
|
||
# own instrument is least likely to be legible. Collecting logs centrally
|
||
# makes the question answerable without host access.
|
||
#
|
||
# It is the collector that feeds this, not the services directly: one ingest
|
||
# point per swarm, same shape as the metrics path.
|
||
#
|
||
# Gateway vhost is authenticated, unlike the metrics store's: an operator
|
||
# reaches this at `https://${cfg.domain}/` gated by the same `auth_request`
|
||
# check against authelia that swarm-ui's own vhost uses (`swarmAuthRequest`
|
||
# below, same shape as `swarm-ui.nix`'s) — see that file's copy for the full
|
||
# rationale (forceSSL is load-bearing there too: authelia answers a plain-http
|
||
# auth subrequest with 400, which `auth_request` cannot interpret as anything
|
||
# but a broken check). The store's own listener stays loopback-only and
|
||
# unauthenticated exactly as before — the collector still writes to it
|
||
# directly, never through this vhost — so this adds a new authenticated front
|
||
# door without touching the existing write path at all.
|
||
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
cfg = config.services.hyperhive.swarm.victorialogs;
|
||
deployCfg = config.services.hyperhive.deploy;
|
||
networkCfg = config.services.hyperhive.network;
|
||
hyperhiveCfg = config.services.hyperhive;
|
||
gatewayCfg = hyperhiveCfg.gateway;
|
||
autheliaCfg = hyperhiveCfg.swarm.authelia;
|
||
swarmDomain = hyperhiveCfg.swarm.domain;
|
||
|
||
# Total on a null swarm domain for the same reason every sibling module is:
|
||
# the required-domain assertion in hive-network.nix should be what an
|
||
# operator sees, not a coercion error from here.
|
||
domainBase = if swarmDomain == null then "invalid" else swarmDomain;
|
||
|
||
# Copy of `swarm-ui.nix`'s own `swarmAuthRequest` — not shared code because
|
||
# this vhost needs exactly the two locations that reference it (`/` and the
|
||
# internal auth-request target below) and pulling in swarm-ui.nix for one
|
||
# string would couple this module to swarm-ui existing at all, which it
|
||
# need not. Same three lines, same reasoning as that file's own comment.
|
||
swarmAuthRequest = ''
|
||
auth_request /__hive_authelia;
|
||
auth_request_set $target_url $scheme://$http_host$request_uri;
|
||
error_page 401 =302 https://${autheliaCfg.domain}/?rd=$target_url;
|
||
'';
|
||
in
|
||
{
|
||
# What stays here is what the store IS from any hive's point of view: its
|
||
# package, the name it answers on, the port. `enable` and `retentionPeriod`
|
||
# are decisions of the host that runs it and live under `deploy.*`.
|
||
options.services.hyperhive.swarm.victorialogs = {
|
||
package = lib.mkOption {
|
||
type = lib.types.package;
|
||
default = pkgs.victorialogs;
|
||
defaultText = lib.literalExpression "pkgs.victorialogs";
|
||
description = "VictoriaLogs package to run.";
|
||
};
|
||
|
||
machine = lib.mkOption {
|
||
type = lib.types.str;
|
||
readOnly = true;
|
||
default = "swarm-victorialogs";
|
||
description = ''
|
||
Container name. Read-only: the name appears in host paths and in
|
||
`machinectl`, so it is a fact other modules may read rather than a
|
||
knob.
|
||
'';
|
||
};
|
||
|
||
domain = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "logs.${domainBase}";
|
||
defaultText = lib.literalExpression ''"logs.''${services.hyperhive.swarm.domain}"'';
|
||
description = ''
|
||
Name the gateway serves this on, behind the same authelia
|
||
`auth_request` gate as the swarm UI's own vhost. A sibling of the
|
||
swarm's other service names, so the swarm-services sub-CA can
|
||
issue for it — see `hive-tls.nix` for why a service name being a
|
||
sibling rather than a child decides which CA may sign it, and
|
||
`swarm.nix`'s `serviceDomains'` for where this name has to be
|
||
registered for that to actually happen.
|
||
'';
|
||
};
|
||
|
||
port = lib.mkOption {
|
||
type = lib.types.port;
|
||
default = 9428;
|
||
description = ''
|
||
Loopback port the store listens on. Upstream's default, kept
|
||
because there is no reason to move it and a familiar number is
|
||
one less thing an operator has to look up.
|
||
|
||
⚠️ Every swarm container shares the host's network namespace, so
|
||
this is a swarm-wide claim rather than a per-container one — two
|
||
modules picking the same number collide at runtime with no bind
|
||
error and nothing in any log. `state/eval-port-collisions.sh`
|
||
checks the class.
|
||
'';
|
||
};
|
||
|
||
};
|
||
|
||
# Retention is a property of the store this host runs, not something the
|
||
# swarm has to agree on: it is read only where the container is defined,
|
||
# and a hive that is a *client* of the log store never consults it. That
|
||
# makes it a `deploy.*` value by the same rule as the seal on the secret
|
||
# store — options on the auto-deployed service itself.
|
||
options.services.hyperhive.deploy.victorialogs.retentionPeriod = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "30d";
|
||
example = "90d";
|
||
description = ''
|
||
How long log data is kept.
|
||
|
||
Deliberately far shorter than the metrics store's retention: logs
|
||
are orders of magnitude larger per unit of time, and their value
|
||
decays much faster. A log line answers "what happened during that
|
||
incident"; a metric answers "is this worse than last quarter".
|
||
'';
|
||
};
|
||
|
||
config = lib.mkIf (hyperhiveCfg.enable && deployCfg.victorialogs.enable) {
|
||
# This store publishes its own health as prometheus metrics on the same
|
||
# listener it serves queries on, so the swarm's collector scrapes it with
|
||
# no exporter and no extra port — same arrangement as the metrics store.
|
||
#
|
||
# Declared here rather than in the collector's module because that is the
|
||
# rule the option carries: an entry exists only where the service that
|
||
# named it runs. ⚠️ That constrains the TARGET, not the scraper — a
|
||
# deployment that splits this container away from the collector's host
|
||
# silently drops the entry, and no assertion can see it, because separate
|
||
# hosts are separate evaluations.
|
||
services.hyperhive.swarm.otel.scrapeTargets.victorialogs = "127.0.0.1:${toString cfg.port}";
|
||
|
||
# The gateway name and the quick-link, both inside `deployCfg.victorialogs.enable` — same
|
||
# "only the host that runs the service may claim the name" guard every
|
||
# sibling swarm-service module uses (`swarm-grafana.nix`,
|
||
# `swarm-victoriametrics.nix`).
|
||
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
||
|
||
# Declared here rather than in the collector, so this store's logs are
|
||
# collected because it runs, not because a list elsewhere remembered it.
|
||
services.hyperhive.swarm.otel.journaldUnits = [ "victorialogs" ];
|
||
|
||
services.hyperhive.swarm.controller.links = [
|
||
{
|
||
label = "Logs";
|
||
icon = "📜";
|
||
url = "https://${cfg.domain}/";
|
||
}
|
||
];
|
||
|
||
# Authenticated front door onto the loopback-only store — see the
|
||
# file-top comment for why this is safe to add without touching the
|
||
# store's own (still unauthenticated, still loopback) listener at all.
|
||
# `removeAttrs`/`forceSSL`: same asymmetry `swarm-ui.nix` documents —
|
||
# authelia answers a plain-http auth subrequest with 400, which
|
||
# `auth_request` cannot read as anything but a broken check, so this
|
||
# vhost needs `forceSSL` rather than the `addSSL` every unauthenticated
|
||
# sibling vhost uses.
|
||
services.nginx.virtualHosts."${cfg.domain}" =
|
||
(builtins.removeAttrs (gatewayCfg.lib.tlsFor cfg.domain) [ "addSSL" ])
|
||
// {
|
||
forceSSL = true;
|
||
listen = gatewayCfg.lib.listen;
|
||
extraConfig = gatewayCfg.lib.securityHeaders;
|
||
locations = {
|
||
"/" = {
|
||
proxyPass = "http://127.0.0.1:${toString cfg.port}/";
|
||
extraConfig = swarmAuthRequest;
|
||
};
|
||
# The subrequest itself — same target, same header set, same
|
||
# reasoning as `swarm-ui.nix`'s own copy (measured against the
|
||
# pinned authelia binary, not copied from an example).
|
||
"= /__hive_authelia" = {
|
||
proxyPass = "https://${autheliaCfg.domain}/api/authz/auth-request";
|
||
# nixpkgs appends its OWN `Host $host` after extraConfig,
|
||
# which would override verifiedProxyTo's — see the comment
|
||
# on verifiedProxyTo in hive-gateway/vhost-lib.nix.
|
||
recommendedProxySettings = false;
|
||
extraConfig = ''
|
||
internal;
|
||
${gatewayCfg.lib.verifiedProxyTo autheliaCfg.domain}
|
||
proxy_pass_request_body off;
|
||
proxy_set_header Content-Length "";
|
||
proxy_set_header X-Original-Method $request_method;
|
||
proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
|
||
proxy_set_header X-Forwarded-Proto $scheme;
|
||
proxy_set_header X-Forwarded-Host $http_host;
|
||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||
'';
|
||
};
|
||
};
|
||
};
|
||
|
||
containers.${cfg.machine} = {
|
||
autoStart = true;
|
||
ephemeral = false;
|
||
# Shared host netns, like every sibling swarm container: the collector
|
||
# and Grafana reach this at 127.0.0.1:<port>.
|
||
privateNetwork = false;
|
||
|
||
config =
|
||
{ ... }:
|
||
{
|
||
imports = [
|
||
(import ./swarm-container-resolver.nix {
|
||
inherit (networkCfg) bridgeIp;
|
||
dnsConsumers = [ "victorialogs.service" ];
|
||
})
|
||
];
|
||
|
||
system.stateVersion = "26.05";
|
||
|
||
# This container shares the host netns, so its own firewall.service
|
||
# would rewrite the HOST ruleset at every boot. The host firewall
|
||
# owns all filtering.
|
||
networking.firewall.enable = false;
|
||
# resolvconf stays off because the resolver unit imported above
|
||
# owns /etc/resolv.conf. Leaving it on would let host-tracking
|
||
# regenerate the file empty, since the host's copy doesn't cross
|
||
# the boundary after start.
|
||
networking.resolvconf.enable = lib.mkForce false;
|
||
|
||
services.victorialogs = {
|
||
enable = true;
|
||
package = cfg.package;
|
||
|
||
# ⚠️ PINNED TO LOOPBACK for the same reason the metrics store is,
|
||
# and it matters more here: upstream's default listens on every
|
||
# interface, and this endpoint accepts writes as well as reads
|
||
# with no authentication of its own. The bind address is the
|
||
# boundary.
|
||
listenAddress = "127.0.0.1:${toString cfg.port}";
|
||
|
||
extraOptions = [ "-retentionPeriod=${deployCfg.victorialogs.retentionPeriod}" ];
|
||
};
|
||
};
|
||
};
|
||
};
|
||
|
||
# 🔑 OTLP ingest is served at `/insert/opentelemetry/v1/logs`, measured
|
||
# against the pinned build rather than read from docs. The path differs
|
||
# from the metrics store's `/opentelemetry/api/v1/push`, so an exporter
|
||
# configured by analogy with the metrics one is silently wrong.
|
||
#
|
||
# ⚠️ And the status code cannot tell you which is which: a wrong path, a
|
||
# right path with a wrong body, and a nonsense path ALL answer 400. Only
|
||
# the server log distinguishes them — the real route complains about the
|
||
# encoding ("json encoding isn't supported ... use protobuf"), while
|
||
# anything else logs "unsupported path requested". Verified with a
|
||
# nonsense-path control, because a probe where every arm returns the same
|
||
# code is not evidence.
|
||
#
|
||
# ⚠️ Like the metrics store's, that endpoint is unauthenticated — which is
|
||
# why `listenAddress` above is loopback, why the collector is the only
|
||
# writer, and why this module declares no gateway vhost.
|
||
}
|