95b89970ee
The socket scheme needs VictoriaMetrics 1.153.0 or later: older builds create the socket 0600 whatever the umask, so nginx could not connect and every consumer would get 502 with the module evaluating clean. deploy.victoriametrics.package is an operator option, so an assertion now refuses an older one, naming the option and the reason. Turning DynamicUser off moves the state directory from /var/lib/private/victoriametrics (behind a /var/lib/victoriametrics symlink) to a plain /var/lib/victoriametrics. A oneshot ordered before the store moves the old directory there and chowns it to the static user when the plain path is the old symlink, empty, or missing, and does nothing once the private directory is gone. Without it the persistent container's history depends on systemd's own DynamicUser-off migration. Refs #4815
352 lines
15 KiB
Nix
352 lines
15 KiB
Nix
# The swarm's metrics store: one VictoriaMetrics for the whole swarm, in a
|
|
# `swarm-victoriametrics` nixos-container.
|
|
#
|
|
# A LOCAL time-series database rather than only an external sink, and that is
|
|
# the point rather than a convenience: the swarm dashboard has to stay
|
|
# readable when the outside world is unreachable. Same failure-domain property
|
|
# the hive-status KV has — a view of the system must not depend on the system
|
|
# it is viewing being healthy.
|
|
#
|
|
# It is the collector that feeds this (the gateway OTEL collector), not the
|
|
# agents directly: one ingest point per swarm, authenticated there.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.swarm.victoriametrics;
|
|
deployCfg = config.services.hyperhive.deploy;
|
|
hyperhiveCfg = config.services.hyperhive;
|
|
gatewayCfg = hyperhiveCfg.gateway;
|
|
|
|
# A netns of its own with no interface but loopback: the store makes no
|
|
# outbound calls, and the gateway reaches it over `socketPath` below, so
|
|
# nothing on the host or the hive bridge can dial its listener.
|
|
privateNetwork = true;
|
|
|
|
# Fixed, not an option: the directory's mode is what keeps the socket
|
|
# private, so it holds this one socket and nothing else.
|
|
socketDir = "/run/swarm-victoriametrics";
|
|
socketPath = "${socketDir}/victoriametrics.sock";
|
|
upstream = path: "http://unix:${socketPath}:${path}";
|
|
|
|
# The gid the gateway's nginx runs as. Only the number crosses into the
|
|
# container, which knows it as `gateway-nginx`.
|
|
nginxGid = config.ids.gids.nginx;
|
|
|
|
# The store's read API: the only part of it the vhost serves without a
|
|
# credential. Each location's `proxy_pass` carries the path, so the store
|
|
# receives the path nginx matched, not the caller's unnormalised spelling.
|
|
#
|
|
# Grafana's Prometheus datasource reads the query, series, label, metadata,
|
|
# buildinfo, exemplar, rules and alerts routes; the store answers the last
|
|
# three with fixed placeholders. vmui is the static files under `/vmui/`
|
|
# plus the query API and the remaining read-only routes here. Paths read off
|
|
# the pinned build's `app/vmselect/main.go` and vmui's `src/`.
|
|
readPaths = [
|
|
"/api/v1/query"
|
|
"/api/v1/query_range"
|
|
"/api/v1/query_exemplars"
|
|
"/api/v1/series"
|
|
"/api/v1/labels"
|
|
"/api/v1/metadata"
|
|
"/api/v1/rules"
|
|
"/api/v1/alerts"
|
|
"/api/v1/status/buildinfo"
|
|
"/api/v1/export"
|
|
"/api/v1/export/csv"
|
|
"/api/v1/status/tsdb"
|
|
"/api/v1/status/metric_names_stats"
|
|
"/api/v1/status/top_queries"
|
|
"/api/v1/status/active_queries"
|
|
"/expand-with-exprs"
|
|
"/prettify-query"
|
|
"/metric-relabel-debug"
|
|
"/vmui"
|
|
];
|
|
# The store serves only `<name>/values` under `/api/v1/label/`, and only
|
|
# static files and UI config under `/vmui/`.
|
|
readPrefixes = [
|
|
"/api/v1/label/"
|
|
"/vmui/"
|
|
];
|
|
readLocations =
|
|
let
|
|
to = path: { proxyPass = upstream path; };
|
|
in
|
|
lib.listToAttrs (
|
|
map (p: lib.nameValuePair "= ${p}" (to p)) readPaths
|
|
++ map (p: lib.nameValuePair "^~ ${p}" (to p)) readPrefixes
|
|
);
|
|
in
|
|
{
|
|
# What the store IS from any hive's point of view, the name it answers on,
|
|
# is `swarm.victoriametrics` in ./swarm-victoriametrics-service.nix.
|
|
# `enable`, `package` and `retentionPeriod` are decisions of the host that
|
|
# runs it and live under `deploy.*`.
|
|
|
|
# Retention is a property of the store this host runs, not something the
|
|
# swarm has to agree on: it is read only where the container is defined,
|
|
# and a hive that is a *client* of the metrics store never consults it.
|
|
options.services.hyperhive.deploy.victoriametrics.package = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = pkgs.victoriametrics;
|
|
defaultText = lib.literalExpression "pkgs.victoriametrics";
|
|
description = "VictoriaMetrics package to run.";
|
|
};
|
|
|
|
options.services.hyperhive.deploy.victoriametrics.retentionPeriod = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "5y";
|
|
example = "90d";
|
|
description = ''
|
|
How long samples are kept.
|
|
|
|
Deliberately a high default rather than a required option: the two
|
|
failure directions are not symmetric. Too long fills a disk, which
|
|
is visible and recoverable by lowering this; too short **destroys
|
|
history**, silently and permanently. So the safe default is generous
|
|
and an operator lowers it once they have measured how fast this swarm
|
|
actually accumulates data.
|
|
'';
|
|
};
|
|
|
|
config = lib.mkIf deployCfg.victoriametrics.enable {
|
|
assertions = [
|
|
{
|
|
assertion = lib.versionAtLeast (lib.getVersion deployCfg.victoriametrics.package) "1.153.0";
|
|
message = ''
|
|
services.hyperhive.deploy.victoriametrics.package must be
|
|
VictoriaMetrics 1.153.0 or later (this one is
|
|
${lib.getVersion deployCfg.victoriametrics.package}).
|
|
|
|
The gateway reaches the store over a unix socket, and only from
|
|
1.153.0 does VictoriaMetrics create that socket under the process
|
|
umask. An older build creates it 0600 whatever the umask, so nginx
|
|
cannot connect and every read, push and scrape fails with 502.
|
|
'';
|
|
}
|
|
];
|
|
|
|
# The gateway name, inside `deployCfg.victoriametrics.enable` — that
|
|
# guard is the load-bearing part. Every hive in a swarm may know this
|
|
# store exists, but only the host that RUNS it may claim the name; a
|
|
# client hive declaring the vhost would answer for a service it does not
|
|
# have.
|
|
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
|
|
|
# This host serves the vhost, and the hive's dnsmasq answers its name.
|
|
services.hyperhive.gateway.enable = lib.mkDefault true;
|
|
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
|
|
|
services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // {
|
|
listen = gatewayCfg.lib.listen;
|
|
extraConfig = gatewayCfg.lib.securityHeaders;
|
|
locations = readLocations // {
|
|
# Nothing reaches the store unless a location here names it. The
|
|
# store serves writes and admin calls (`/api/v1/import`,
|
|
# `/api/v1/admin/tsdb/delete_series`, ...) on the same listener as
|
|
# its queries, and authenticates none of them.
|
|
"/".return = "403";
|
|
|
|
# The swarm UI's quick link opens the bare domain. The store's own
|
|
# index page links mostly to paths refused above, so send it to vmui.
|
|
"= /".return = "302 /vmui/";
|
|
|
|
# The swarm collector's ingest route. `=` so it outranks the `/`
|
|
# prefix above.
|
|
#
|
|
# ⚠️ No `error_page 401 =302` here, and its absence is the point: a
|
|
# redirect is right for a browser and wrong for a pusher, which would
|
|
# follow it and POST its batch at a login page that answers 200 —
|
|
# ingest reporting healthy while storing nothing.
|
|
"= /opentelemetry/api/v1/push" = {
|
|
proxyPass = upstream "/opentelemetry/api/v1/push";
|
|
extraConfig = ''
|
|
auth_request /__metrics_push_authz;
|
|
'';
|
|
};
|
|
|
|
# The store's own health, for the swarm collector wherever it runs:
|
|
# served on the query listener, so no exporter and no extra port. `=`
|
|
# for the same reason as the push route, and no `error_page` for the
|
|
# same reason: a scraper would parse a login page as metrics.
|
|
#
|
|
# The collector requests exactly this URL, declared as
|
|
# `publishedScrapeTargets.victoriametrics` in ./swarm-otel-service.nix;
|
|
# the two must agree, since that string is also the token's audience.
|
|
"= /metrics" = {
|
|
proxyPass = upstream "/metrics";
|
|
extraConfig = ''
|
|
auth_request /__metrics_push_authz;
|
|
'';
|
|
};
|
|
|
|
# The subrequest for both gated locations above. Authelia admits the
|
|
# collector's client anywhere on this name and refuses every other
|
|
# subject (`metricsStoreRules` in ./swarm-authelia.nix), so the scrape
|
|
# needs no subrequest of its own. Same target and header set as the
|
|
# sibling log store's, which took them from `swarm-ui.nix` —
|
|
# `X-Original-URL` and `X-Original-Method` are what authelia's
|
|
# auth-request implementation reads, and the address it compares the
|
|
# token's audience against.
|
|
"= /__metrics_push_authz" = {
|
|
proxyPass = "https://${hyperhiveCfg.swarm.authelia.domain}/api/authz/auth-request";
|
|
# nixpkgs appends its OWN `Host $host` after extraConfig, which
|
|
# would override verifiedProxyTo's — see the comment on
|
|
# verifiedProxyTo in hive-gateway/vhost-lib.nix.
|
|
recommendedProxySettings = false;
|
|
extraConfig = ''
|
|
internal;
|
|
${gatewayCfg.lib.verifiedProxyTo hyperhiveCfg.swarm.authelia.domain}
|
|
proxy_pass_request_body off;
|
|
proxy_set_header Content-Length "";
|
|
proxy_set_header X-Original-Method $request_method;
|
|
proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
|
|
proxy_set_header X-Forwarded-Proto $scheme;
|
|
proxy_set_header X-Forwarded-Host $http_host;
|
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
|
'';
|
|
};
|
|
};
|
|
};
|
|
|
|
# The socket directory, created host-side before the container starts:
|
|
# nixos-container refuses to start when a bind source is missing.
|
|
#
|
|
# Root-owned and group nginx, `0770`: the store creates its socket here
|
|
# through the group (its uid is allocated inside the container, so the
|
|
# host cannot name it), and the gateway traverses it the same way.
|
|
# Everything else is kept out.
|
|
systemd.tmpfiles.rules = [
|
|
"d ${socketDir} 0770 root ${toString nginxGid} - -"
|
|
];
|
|
|
|
containers.${cfg.machine} = {
|
|
autoStart = true;
|
|
ephemeral = false;
|
|
# Journal files on the host, not inside the container: nixpkgs hardcodes
|
|
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
|
|
extraFlags = [ "--link-journal=host" ];
|
|
inherit privateNetwork;
|
|
|
|
# A bind mount rather than a path both sides happen to know: a
|
|
# container's `/run` is an nspawn tmpfs the host cannot see into.
|
|
bindMounts.${socketDir} = {
|
|
hostPath = socketDir;
|
|
isReadOnly = false;
|
|
};
|
|
|
|
config =
|
|
{ config, pkgs, ... }:
|
|
{
|
|
imports = [ ../container-modules/swarm-container.nix ];
|
|
|
|
# No resolver unit: with no interface but loopback there is no
|
|
# nameserver to reach, and the store resolves no names.
|
|
services.hyperhive.swarmContainer = {
|
|
inherit privateNetwork;
|
|
writesOwnResolvConf = false;
|
|
};
|
|
|
|
# The socket's group is the store's primary group, so that group is
|
|
# nginx's gid. A static user, not nixpkgs' DynamicUser: a dynamic
|
|
# user's group is allocated per start, and nginx is never in it.
|
|
users.groups.gateway-nginx.gid = nginxGid;
|
|
users.users.victoriametrics = {
|
|
isSystemUser = true;
|
|
group = "gateway-nginx";
|
|
};
|
|
|
|
systemd.services.victoriametrics = {
|
|
serviceConfig = {
|
|
DynamicUser = lib.mkForce false;
|
|
User = "victoriametrics";
|
|
Group = "gateway-nginx";
|
|
# The store creates its socket under the process umask, so
|
|
# `0007` makes it `0770` and nginx's group can connect.
|
|
UMask = "0007";
|
|
};
|
|
# nixpkgs' readiness loop curls `http://<listenAddress>/ping`,
|
|
# which no `unix:` address answers.
|
|
postStart = lib.mkForce ''
|
|
until ${lib.getExe pkgs.curl} -s -o /dev/null --unix-socket ${socketPath} http://localhost/ping; do
|
|
sleep 1;
|
|
done
|
|
'';
|
|
};
|
|
|
|
# Moves the store's history out of the DynamicUser layout, where
|
|
# `/var/lib/<stateDir>` is a symlink into `/var/lib/private/`, to the
|
|
# plain directory the static user's StateDirectory is. Runs before
|
|
# the store on every start and does nothing once
|
|
# `/var/lib/private/<stateDir>` is gone. When both paths hold data
|
|
# it moves neither and the store starts on the plain one.
|
|
systemd.services.victoriametrics-state-migrate =
|
|
let
|
|
old = "/var/lib/private/${config.services.victoriametrics.stateDir}";
|
|
new = "/var/lib/${config.services.victoriametrics.stateDir}";
|
|
in
|
|
{
|
|
description = "move VictoriaMetrics' DynamicUser state directory to its static path";
|
|
before = [ "victoriametrics.service" ];
|
|
requiredBy = [ "victoriametrics.service" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
SyslogIdentifier = "victoriametrics-state-migrate";
|
|
};
|
|
path = [ pkgs.coreutils ];
|
|
script = ''
|
|
set -eu
|
|
old=${lib.escapeShellArg old}
|
|
new=${lib.escapeShellArg new}
|
|
if [ ! -d "$old" ] || [ -L "$old" ]; then
|
|
exit 0
|
|
fi
|
|
if [ -L "$new" ]; then
|
|
rm "$new"
|
|
elif [ -d "$new" ]; then
|
|
if [ -n "$(ls -A "$new")" ]; then
|
|
echo "victoriametrics-state-migrate: $old and $new both hold data; moving neither" >&2
|
|
exit 0
|
|
fi
|
|
rmdir "$new"
|
|
fi
|
|
mv "$old" "$new"
|
|
chown -R victoriametrics:gateway-nginx "$new"
|
|
echo "victoriametrics-state-migrate: moved $old to $new"
|
|
'';
|
|
};
|
|
|
|
services.victoriametrics = {
|
|
enable = true;
|
|
package = deployCfg.victoriametrics.package;
|
|
retentionPeriod = deployCfg.victoriametrics.retentionPeriod;
|
|
|
|
# ⚠️ A socket, not upstream's default `:8428` on every interface:
|
|
# the gateway is the only intended client, and this listener
|
|
# carries an unauthenticated write endpoint (see the OTLP note
|
|
# below).
|
|
listenAddress = "unix:${socketPath}";
|
|
};
|
|
};
|
|
};
|
|
};
|
|
|
|
# 🔑 OTLP ingest needs no flag. Measured against the pinned 1.146.0 rather
|
|
# than inferred from the module's option list, which has no OTLP switch and
|
|
# so reads as though the feature were absent: the running server answers
|
|
# `POST /opentelemetry/api/v1/push` with 200 (a nonexistent path answers
|
|
# 400, so that 200 means the route exists). The `-opentelemetry.*` flags
|
|
# only tune naming and limits, and are reachable via `extraOptions` if a
|
|
# deployment ever needs them.
|
|
#
|
|
# ⚠️ That endpoint is unauthenticated, which is why `listenAddress` above is
|
|
# a socket only the gateway can reach: the only route to it is the vhost's
|
|
# ingest location, which authenticates. The collector is still the only writer, but
|
|
# it now arrives by the swarm name rather than over loopback, because it need
|
|
# not share a host with this store.
|
|
}
|