vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity
hyperhive/nix/host-modules/swarm-victoriametrics.nix
atlas ddb583cc22 gateway: service modules declare their vhost under gateway.vhosts 
Ten service modules (forge, matrix chat, authelia, grafana, otel, nats,
swarm-ui, victorialogs, victoriametrics, bao UI) wrote
services.nginx.virtualHosts themselves, each applying the gateway's
listen set, per-name TLS attrs and security headers by hand. They now
declare services.hyperhive.gateway.vhosts.<name> with only their own
locations and options; ./declared-vhosts.nix renders it into the nginx
vhost with listen, TLS and headers added.

An eval assertion refuses a declared vhost that sets listen, addSSL,
onlySSL, enableACME, useACMEHost, sslCertificate or sslCertificateKey,
and one declared while the gateway is off. nats' own leaf is the new
`certificate` field. The rendered nginx vhosts are unchanged.
2026-10-16 12:47:03 +02:00

550 lines
23 KiB
Nix
Raw Blame History

# The swarm's metrics store: one VictoriaMetrics for the whole swarm, in a
# `swarm-victoriametrics` nixos-container.
#
# A LOCAL time-series database rather than only an external sink, and that is
# the point rather than a convenience: the swarm dashboard has to stay
# readable when the outside world is unreachable. Same failure-domain property
# the hive-status KV has — a view of the system must not depend on the system
# it is viewing being healthy.
#
# It is the collector that feeds this (the gateway OTEL collector), not the
# agents directly: one ingest point per swarm, authenticated there.
{
pkgs,
lib,
config,
...
}:
let
cfg = config.services.hyperhive.swarm.victoriametrics;
deployCfg = config.services.hyperhive.deploy;
hyperhiveCfg = config.services.hyperhive;
gatewayCfg = hyperhiveCfg.gateway;
# A netns of its own with no interface but loopback: the store makes no
# outbound calls, and the gateway reaches it over `socketPath` below, so
# nothing on the host or the hive bridge can dial its listener.
privateNetwork = true;
# Fixed, not an option: the directory's mode is what keeps the socket
# private, so it holds this one socket and nothing else.
socketDir = "/run/swarm-victoriametrics";
socketPath = "${socketDir}/victoriametrics.sock";
upstream = path: "http://unix:${socketPath}:${path}";
# The gid the gateway's nginx runs as. Only the number crosses into the
# container, which knows it as `gateway-nginx`.
nginxGid = config.ids.gids.nginx;
# The store's read API, which the vhost forwards to a caller holding
# Grafana's bearer or an operator's session. Each location's `proxy_pass`
# carries the path, so the store receives the path nginx matched, not the
# caller's unnormalised spelling.
#
# Grafana's Prometheus datasource reads the query, series, label, metadata,
# buildinfo, exemplar, rules and alerts routes; the store answers the last
# three with fixed placeholders. vmui (`uiLocations` below) is static files
# plus the query API and the remaining read-only routes here. Paths read off
# the pinned build's `app/vmselect/main.go` and vmui's `src/`.
readPaths = [
"/api/v1/query"
"/api/v1/query_range"
"/api/v1/query_exemplars"
"/api/v1/series"
"/api/v1/labels"
"/api/v1/metadata"
"/api/v1/rules"
"/api/v1/alerts"
"/api/v1/status/buildinfo"
"/api/v1/export"
"/api/v1/export/csv"
"/api/v1/status/tsdb"
"/api/v1/status/metric_names_stats"
"/api/v1/status/top_queries"
"/api/v1/status/active_queries"
"/expand-with-exprs"
"/prettify-query"
"/metric-relabel-debug"
];
# The store serves only `<name>/values` under `/api/v1/label/`.
readPrefixes = [ "/api/v1/label/" ];
# Bare `auth_request`, no login redirect: a machine caller handed
# `error_page 401 =302` follows it and reads authelia's login page as a 200
# holding no series. `/__metrics_read_auth` picks the check from the
# caller's `Authorization` (`appendHttpConfig` below).
readAuth = ''
auth_request /__metrics_read_auth;
'';
readLocations =
let
to = path: {
proxyPass = upstream path;
extraConfig = readAuth;
};
in
lib.listToAttrs (
map (p: lib.nameValuePair "= ${p}" (to p)) readPaths
++ map (p: lib.nameValuePair "^~ ${p}" (to p)) readPrefixes
);
# The same three lines as `swarmAuthRequest` in ./swarm-victorialogs.nix,
# for the same browser: an operator without a session is sent to log in.
swarmAuthRequest = ''
auth_request /__metrics_authelia;
auth_request_set $target_url $scheme://$http_host$request_uri;
error_page 401 =302 https://${hyperhiveCfg.swarm.authelia.domain}/?rd=$target_url;
'';
# vmui's static files and UI config. Only a browser loads these, and its
# queries then go to the read locations above with the same session cookie.
uiLocations = {
"= /vmui" = {
proxyPass = upstream "/vmui";
extraConfig = swarmAuthRequest;
};
"^~ /vmui/" = {
proxyPass = upstream "/vmui/";
extraConfig = swarmAuthRequest;
};
};
# The `auth_request` responder for Grafana's bearer (`swarm-metrics-auth`).
# Its socket's directory holds that socket alone and is traverse-only for
# others, the shape `swarm-logs-auth` in ./swarm-victorialogs.nix has.
metricsAuthUnit = "swarm-metrics-auth";
metricsAuthSocket = "/run/${metricsAuthUnit}/auth.sock";
metricsAuthCredential = id: "/run/credentials/${metricsAuthUnit}.service/${id}";
# Credential id → host path of the responder's store identity.
metricsAuthStoreFiles = {
"bao-client.pem" = deployCfg.victoriametrics.authBaoClientCertFile;
"bao-client-key.pem" = deployCfg.victoriametrics.authBaoClientKeyFile;
}
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
"bao-ca.pem" = deployCfg.bao.serverCaFile;
};
baoCfg = hyperhiveCfg.swarm.bao;
# The path Grafana's host creates the bearer at, under the KV mount. The
# same leaf as `grafanaMetricsBearerLeaf` in ./swarm-bao.nix.
grafanaBearerStorePath = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer";
in
{
# What the store IS from any hive's point of view, the name it answers on,
# is `swarm.victoriametrics` in ./swarm-victoriametrics-service.nix.
# `enable`, `package` and `retentionPeriod` are decisions of the host that
# runs it and live under `deploy.*`.
# Retention is a property of the store this host runs, not something the
# swarm has to agree on: it is read only where the container is defined,
# and a hive that is a *client* of the metrics store never consults it.
options.services.hyperhive.deploy.victoriametrics.package = lib.mkOption {
type = lib.types.package;
default = pkgs.victoriametrics;
defaultText = lib.literalExpression "pkgs.victoriametrics";
description = "VictoriaMetrics package to run.";
};
options.services.hyperhive.deploy.victoriametrics.retentionPeriod = lib.mkOption {
type = lib.types.str;
default = "5y";
example = "90d";
description = ''
How long samples are kept.
Deliberately a high default rather than a required option: the two
failure directions are not symmetric. Too long fills a disk, which
is visible and recoverable by lowering this; too short **destroys
history**, silently and permanently. So the safe default is generous
and an operator lowers it once they have measured how fast this swarm
actually accumulates data.
'';
};
options.services.hyperhive.deploy.victoriametrics.authPackage = lib.mkOption {
type = lib.types.package;
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-metrics-auth";
description = ''
The `swarm-metrics-auth` package: the gateway's `auth_request` responder
that checks the bearer Grafana's metrics datasource presents on the
store's query API against the secret store.
'';
};
options.services.hyperhive.deploy.victoriametrics.authBaoClientCertFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "/var/lib/swarm-bao-pki/metrics-auth.pem";
description = ''
Client certificate `swarm-metrics-auth` presents to the swarm's secret
store to read Grafana's metrics bearer. Its subject must be
{option}`services.hyperhive.deploy.bao.metricsAuthCommonName`.
Required when `deploy.victoriametrics.enable` is set; unset fails at
eval. ./glue-metrics-auth-bao-identity.nix points it at the leaf
./glue-bao-tls.nix mints on a host that runs the store; on any other
metrics-store host it is a file the operator places.
A path, never a value.
'';
};
options.services.hyperhive.deploy.victoriametrics.authBaoClientKeyFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "/var/lib/swarm-bao-pki/metrics-auth-key.pem";
description = ''
Private key for {option}`services.hyperhive.deploy.victoriametrics.authBaoClientCertFile`.
A path, never a value.
'';
};
config = lib.mkIf deployCfg.victoriametrics.enable {
assertions = [
{
assertion = lib.versionAtLeast (lib.getVersion deployCfg.victoriametrics.package) "1.153.0";
message = ''
services.hyperhive.deploy.victoriametrics.package must be
VictoriaMetrics 1.153.0 or later (this one is
${lib.getVersion deployCfg.victoriametrics.package}).
The gateway reaches the store over a unix socket, and only from
1.153.0 does VictoriaMetrics create that socket under the process
umask. An older build creates it 0600 whatever the umask, so nginx
cannot connect and every read, push and scrape fails with 502.
'';
}
{
assertion =
deployCfg.victoriametrics.authBaoClientCertFile != null
&& deployCfg.victoriametrics.authBaoClientKeyFile != null;
message = ''
services.hyperhive.deploy.victoriametrics.authBaoClientCertFile and
authBaoClientKeyFile must both be set.
swarm-metrics-auth reads Grafana's metrics bearer from the swarm's
secret store with this certificate. They default to the leaf the
store mints where services.hyperhive.deploy.bao.enable is set on
this host; elsewhere, issue a leaf whose CN is
services.hyperhive.deploy.bao.metricsAuthCommonName and name it in
both options.
'';
}
];
# The gateway name, inside `deployCfg.victoriametrics.enable` — that
# guard is the load-bearing part. Every hive in a swarm may know this
# store exists, but only the host that RUNS it may claim the name; a
# client hive declaring the vhost would answer for a service it does not
# have.
services.hyperhive.gateway.localNames = [ cfg.domain ];
# This host serves the vhost, and the hive's dnsmasq answers its name.
services.hyperhive.gateway.enable = lib.mkDefault true;
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
# Which check a read location's `auth_request` gets. `auth_request` takes a
# fixed URI, so it names `/__metrics_read_auth`, which rewrites to the
# target this map picks: a bearer goes to Grafana's check, anything else
# (an operator's session cookie) to authelia.
services.nginx.appendHttpConfig = ''
map $http_authorization $swarm_metrics_read_auth_target {
"~^Bearer " /__metrics_grafana_auth;
default /__metrics_authelia;
}
'';
# `forceSSL`: authelia answers a plain-http auth subrequest
# with 400, which `auth_request` cannot read as anything but a broken
# check. Same as ./swarm-victorialogs.nix's vhost.
services.hyperhive.gateway.vhosts."${cfg.domain}" = {
forceSSL = true;
locations =
readLocations
// uiLocations
// {
# Nothing reaches the store unless a location here names it. The
# store serves writes and admin calls (`/api/v1/import`,
# `/api/v1/admin/tsdb/delete_series`, ...) on the same listener as
# its queries, and authenticates none of them.
"/".return = "403";
# The swarm UI's quick link opens the bare domain. The store's own
# index page links mostly to paths refused above, so send it to vmui.
"= /".return = "302 /vmui/";
# The swarm collector's ingest route. `=` so it outranks the `/`
# prefix above.
#
# ⚠️ No `error_page 401 =302` here, and its absence is the point: a
# redirect is right for a browser and wrong for a pusher, which would
# follow it and POST its batch at a login page that answers 200 —
# ingest reporting healthy while storing nothing.
"= /opentelemetry/api/v1/push" = {
proxyPass = upstream "/opentelemetry/api/v1/push";
extraConfig = ''
auth_request /__metrics_authelia;
'';
};
# The store's own health, for the swarm collector wherever it runs:
# served on the query listener, so no exporter and no extra port. `=`
# for the same reason as the push route, and no `error_page` for the
# same reason: a scraper would parse a login page as metrics.
#
# The collector requests exactly this URL, declared as
# `publishedScrapeTargets.victoriametrics` in ./swarm-otel-service.nix;
# the two must agree, since that string is also the token's audience.
"= /metrics" = {
proxyPass = upstream "/metrics";
extraConfig = ''
auth_request /__metrics_authelia;
'';
};
# `rewrite … last` re-matches locations, so the subrequest lands in
# one of the two internal locations named by the map.
"= /__metrics_read_auth" = {
extraConfig = ''
internal;
rewrite ^ $swarm_metrics_read_auth_target last;
'';
};
# Grafana's bearer, checked by `swarm-metrics-auth`. The
# `Authorization` header rides the subrequest unchanged. No rate limit
# in front: the responder reads the store at most once per its cache
# TTL, whatever the bearers.
"= /__metrics_grafana_auth" = {
proxyPass = "http://unix:${metricsAuthSocket}:/";
extraConfig = ''
internal;
proxy_pass_request_body off;
proxy_set_header Content-Length "";
'';
};
# The authelia subrequest. For the collector's two locations above it
# admits only the collector's client; for the read and vmui locations
# it admits an operator's session (`metricsStoreRules` in
# ./swarm-authelia.nix). Same target and header set as the sibling log
# store's, which took them from `swarm-ui.nix` — `X-Original-URL` and
# `X-Original-Method` are what authelia's auth-request implementation
# reads, and the address it compares the token's audience against.
"= /__metrics_authelia" = {
proxyPass = "https://${hyperhiveCfg.swarm.authelia.domain}/api/authz/auth-request";
# nixpkgs appends its OWN `Host $host` after extraConfig, which
# would override verifiedProxyTo's — see the comment on
# verifiedProxyTo in hive-gateway/vhost-lib.nix.
recommendedProxySettings = false;
extraConfig = ''
internal;
${gatewayCfg.lib.verifiedProxyTo hyperhiveCfg.swarm.authelia.domain}
proxy_pass_request_body off;
proxy_set_header Content-Length "";
proxy_set_header X-Original-Method $request_method;
proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
'';
};
};
};
# The socket directory, created host-side before the container starts:
# nixos-container refuses to start when a bind source is missing.
#
# Root-owned and group nginx, `0770`: the store creates its socket here
# through the group (its uid is allocated inside the container, so the
# host cannot name it), and the gateway traverses it the same way.
# Everything else is kept out.
systemd.tmpfiles.rules = [
"d ${socketDir} 0770 root ${toString nginxGid} - -"
];
# Grafana's check on the read API. On the host rather than in the store's
# container: it needs nothing of the store, only nginx's reach to its
# socket and its own store identity.
users.users.${metricsAuthUnit} = {
isSystemUser = true;
group = metricsAuthUnit;
description = "swarm metrics store auth_request responder";
};
users.groups.${metricsAuthUnit} = { };
systemd.services.${metricsAuthUnit} = {
description = "swarm metrics store auth_request responder for Grafana's bearer";
wantedBy = [ "multi-user.target" ];
# Where the store's PKI is minted on this host, the leaf is one of its
# files. Ordering only: elsewhere the unit does not exist.
after = [
"network.target"
"swarm-bao-pki.service"
];
serviceConfig = {
ExecStart = lib.concatStringsSep " " [
"${deployCfg.victoriametrics.authPackage}/bin/swarm-metrics-auth"
"--socket ${metricsAuthSocket}"
# The role ./swarm-bao.nix writes on the store's `cert` mount.
"--store-cert-role swarm-metrics-auth"
"--store-path ${grafanaBearerStorePath}"
];
# Paths only: `argv` is world-readable via /proc/<pid>/cmdline.
LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") metricsAuthStoreFiles;
# ⚠️ A static user, never `DynamicUser`: with `RuntimeDirectoryPreserve`
# systemd puts a dynamic user's runtime directory under
# `/run/private/` (0700 root:root), which nginx cannot traverse.
User = metricsAuthUnit;
Group = metricsAuthUnit;
RuntimeDirectory = metricsAuthUnit;
# Traverse-only for others, so nginx reaches the socket path.
RuntimeDirectoryMode = "0751";
# The path never vanishes from under a running nginx; the responder
# unlinks a stale socket as it starts.
RuntimeDirectoryPreserve = "yes";
Restart = "on-failure";
RestartSec = "5s";
SyslogIdentifier = metricsAuthUnit;
ProtectSystem = "strict";
ProtectHome = true;
PrivateTmp = "disconnected";
PrivateDevices = true;
NoNewPrivileges = true;
RestrictSUIDSGID = true;
RemoveIPC = true;
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = metricsAuthCredential "bao-client.pem";
BAO_CLIENT_KEY = metricsAuthCredential "bao-client-key.pem";
}
// lib.optionalAttrs (metricsAuthStoreFiles ? "bao-ca.pem") {
BAO_CACERT = metricsAuthCredential "bao-ca.pem";
};
};
containers.${cfg.machine} = {
autoStart = true;
ephemeral = false;
# Journal files on the host, not inside the container: nixpkgs hardcodes
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
extraFlags = [ "--link-journal=host" ];
inherit privateNetwork;
# A bind mount rather than a path both sides happen to know: a
# container's `/run` is an nspawn tmpfs the host cannot see into.
bindMounts.${socketDir} = {
hostPath = socketDir;
isReadOnly = false;
};
config =
{ config, pkgs, ... }:
{
imports = [ ../container-modules/swarm-container.nix ];
# No resolver unit: with no interface but loopback there is no
# nameserver to reach, and the store resolves no names.
services.hyperhive.swarmContainer = {
inherit privateNetwork;
writesOwnResolvConf = false;
};
# The socket's group is the store's primary group, so that group is
# nginx's gid. A static user, not nixpkgs' DynamicUser: a dynamic
# user's group is allocated per start, and nginx is never in it.
users.groups.gateway-nginx.gid = nginxGid;
users.users.victoriametrics = {
isSystemUser = true;
group = "gateway-nginx";
};
systemd.services.victoriametrics = {
serviceConfig = {
DynamicUser = lib.mkForce false;
User = "victoriametrics";
Group = "gateway-nginx";
# The store creates its socket under the process umask, so
# `0007` makes it `0770` and nginx's group can connect.
UMask = "0007";
};
# nixpkgs' readiness loop curls `http://<listenAddress>/ping`,
# which no `unix:` address answers.
postStart = lib.mkForce ''
until ${lib.getExe pkgs.curl} -s -o /dev/null --unix-socket ${socketPath} http://localhost/ping; do
sleep 1;
done
'';
};
# Moves the store's history out of the DynamicUser layout, where
# `/var/lib/<stateDir>` is a symlink into `/var/lib/private/`, to the
# plain directory the static user's StateDirectory is. Runs before
# the store on every start and does nothing once
# `/var/lib/private/<stateDir>` is gone. When both paths hold data
# it moves neither and the store starts on the plain one.
systemd.services.victoriametrics-state-migrate =
let
old = "/var/lib/private/${config.services.victoriametrics.stateDir}";
new = "/var/lib/${config.services.victoriametrics.stateDir}";
in
{
description = "move VictoriaMetrics' DynamicUser state directory to its static path";
before = [ "victoriametrics.service" ];
requiredBy = [ "victoriametrics.service" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
SyslogIdentifier = "victoriametrics-state-migrate";
};
path = [ pkgs.coreutils ];
script = ''
set -eu
old=${lib.escapeShellArg old}
new=${lib.escapeShellArg new}
if [ ! -d "$old" ] || [ -L "$old" ]; then
exit 0
fi
if [ -L "$new" ]; then
rm "$new"
elif [ -d "$new" ]; then
if [ -n "$(ls -A "$new")" ]; then
echo "victoriametrics-state-migrate: $old and $new both hold data; moving neither" >&2
exit 0
fi
rmdir "$new"
fi
mv "$old" "$new"
chown -R victoriametrics:gateway-nginx "$new"
echo "victoriametrics-state-migrate: moved $old to $new"
'';
};
services.victoriametrics = {
enable = true;
package = deployCfg.victoriametrics.package;
retentionPeriod = deployCfg.victoriametrics.retentionPeriod;
# ⚠️ A socket, not upstream's default `:8428` on every interface:
# the gateway is the only intended client, and this listener
# carries an unauthenticated write endpoint (see the OTLP note
# below).
listenAddress = "unix:${socketPath}";
};
};
};
};
# 🔑 OTLP ingest needs no flag. Measured against the pinned 1.146.0 rather
# than inferred from the module's option list, which has no OTLP switch and
# so reads as though the feature were absent: the running server answers
# `POST /opentelemetry/api/v1/push` with 200 (a nonexistent path answers
# 400, so that 200 means the route exists). The `-opentelemetry.*` flags
# only tune naming and limits, and are reachable via `extraOptions` if a
# deployment ever needs them.
#
# ⚠️ That endpoint is unauthenticated, which is why `listenAddress` above is
# a socket only the gateway can reach: the only route to it is the vhost's
# ingest location, which authenticates. The collector is still the only writer, but
# it now arrives by the swarm name rather than over loopback, because it need
# not share a host with this store.
}
View git blame Copy permalink