6aab2ebb65
The metrics vhost forwarded the store's read API (query, series, labels,
export, vmui, ...) with no credential, so anything that could reach
metrics.<swarm> could read every hive's and agent's metrics. mara ruled on
checked by an auth_request responder, and operators use their authelia
session.
- swarm-metrics-auth (new crate): auth_request responder on the metrics
store's host. Compares the presented bearer, in constant time, with the
one stored at swarm/services/<grafana client id>/metrics/bearer, read
under its own bao leaf and cached for 60s. Static user, the post-#5264
swarm-logs-auth unit shape.
- swarm-bao-grafana-metrics-bearer: on Grafana's host, creates that bearer
in bao if absent (create without update, Grafana's existing leaf) and
writes "Bearer <value>" into the container; the datasource sends it as
its Authorization header via $__file{}.
- The vhost: read locations take auth_request through a map (a bearer goes
to swarm-metrics-auth, anything else to authelia, no login redirect);
vmui takes an authelia session with the login redirect; forceSSL like
the logs vhost. The authelia subrequest location is renamed from
/__metrics_push_authz to /__metrics_authelia since it now serves reads.
- Authelia: the collector keeps every path; everyone else is refused the
collector's push and /metrics paths; group admins is admitted to the
rest; then deny.
- bao: a swarm-metrics-auth cert role and policy (read on the one leaf),
its leaf minted by glue-bao-tls, the CN reserved as a hive name, and
Grafana's grant gains create on the bearer leaf.
Fixes #3596
554 lines
24 KiB
Nix
554 lines
24 KiB
Nix
# The swarm's metrics store: one VictoriaMetrics for the whole swarm, in a
|
|
# `swarm-victoriametrics` nixos-container.
|
|
#
|
|
# A LOCAL time-series database rather than only an external sink, and that is
|
|
# the point rather than a convenience: the swarm dashboard has to stay
|
|
# readable when the outside world is unreachable. Same failure-domain property
|
|
# the hive-status KV has — a view of the system must not depend on the system
|
|
# it is viewing being healthy.
|
|
#
|
|
# It is the collector that feeds this (the gateway OTEL collector), not the
|
|
# agents directly: one ingest point per swarm, authenticated there.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.swarm.victoriametrics;
|
|
deployCfg = config.services.hyperhive.deploy;
|
|
hyperhiveCfg = config.services.hyperhive;
|
|
gatewayCfg = hyperhiveCfg.gateway;
|
|
|
|
# A netns of its own with no interface but loopback: the store makes no
|
|
# outbound calls, and the gateway reaches it over `socketPath` below, so
|
|
# nothing on the host or the hive bridge can dial its listener.
|
|
privateNetwork = true;
|
|
|
|
# Fixed, not an option: the directory's mode is what keeps the socket
|
|
# private, so it holds this one socket and nothing else.
|
|
socketDir = "/run/swarm-victoriametrics";
|
|
socketPath = "${socketDir}/victoriametrics.sock";
|
|
upstream = path: "http://unix:${socketPath}:${path}";
|
|
|
|
# The gid the gateway's nginx runs as. Only the number crosses into the
|
|
# container, which knows it as `gateway-nginx`.
|
|
nginxGid = config.ids.gids.nginx;
|
|
|
|
# The store's read API, which the vhost forwards to a caller holding
|
|
# Grafana's bearer or an operator's session. Each location's `proxy_pass`
|
|
# carries the path, so the store receives the path nginx matched, not the
|
|
# caller's unnormalised spelling.
|
|
#
|
|
# Grafana's Prometheus datasource reads the query, series, label, metadata,
|
|
# buildinfo, exemplar, rules and alerts routes; the store answers the last
|
|
# three with fixed placeholders. vmui (`uiLocations` below) is static files
|
|
# plus the query API and the remaining read-only routes here. Paths read off
|
|
# the pinned build's `app/vmselect/main.go` and vmui's `src/`.
|
|
readPaths = [
|
|
"/api/v1/query"
|
|
"/api/v1/query_range"
|
|
"/api/v1/query_exemplars"
|
|
"/api/v1/series"
|
|
"/api/v1/labels"
|
|
"/api/v1/metadata"
|
|
"/api/v1/rules"
|
|
"/api/v1/alerts"
|
|
"/api/v1/status/buildinfo"
|
|
"/api/v1/export"
|
|
"/api/v1/export/csv"
|
|
"/api/v1/status/tsdb"
|
|
"/api/v1/status/metric_names_stats"
|
|
"/api/v1/status/top_queries"
|
|
"/api/v1/status/active_queries"
|
|
"/expand-with-exprs"
|
|
"/prettify-query"
|
|
"/metric-relabel-debug"
|
|
];
|
|
# The store serves only `<name>/values` under `/api/v1/label/`.
|
|
readPrefixes = [ "/api/v1/label/" ];
|
|
|
|
# Bare `auth_request`, no login redirect: a machine caller handed
|
|
# `error_page 401 =302` follows it and reads authelia's login page as a 200
|
|
# holding no series. `/__metrics_read_auth` picks the check from the
|
|
# caller's `Authorization` (`appendHttpConfig` below).
|
|
readAuth = ''
|
|
auth_request /__metrics_read_auth;
|
|
'';
|
|
readLocations =
|
|
let
|
|
to = path: {
|
|
proxyPass = upstream path;
|
|
extraConfig = readAuth;
|
|
};
|
|
in
|
|
lib.listToAttrs (
|
|
map (p: lib.nameValuePair "= ${p}" (to p)) readPaths
|
|
++ map (p: lib.nameValuePair "^~ ${p}" (to p)) readPrefixes
|
|
);
|
|
|
|
# The same three lines as `swarmAuthRequest` in ./swarm-victorialogs.nix,
|
|
# for the same browser: an operator without a session is sent to log in.
|
|
swarmAuthRequest = ''
|
|
auth_request /__metrics_authelia;
|
|
auth_request_set $target_url $scheme://$http_host$request_uri;
|
|
error_page 401 =302 https://${hyperhiveCfg.swarm.authelia.domain}/?rd=$target_url;
|
|
'';
|
|
# vmui's static files and UI config. Only a browser loads these, and its
|
|
# queries then go to the read locations above with the same session cookie.
|
|
uiLocations = {
|
|
"= /vmui" = {
|
|
proxyPass = upstream "/vmui";
|
|
extraConfig = swarmAuthRequest;
|
|
};
|
|
"^~ /vmui/" = {
|
|
proxyPass = upstream "/vmui/";
|
|
extraConfig = swarmAuthRequest;
|
|
};
|
|
};
|
|
|
|
# The `auth_request` responder for Grafana's bearer (`swarm-metrics-auth`).
|
|
# Its socket's directory holds that socket alone and is traverse-only for
|
|
# others, the shape `swarm-logs-auth` in ./swarm-victorialogs.nix has.
|
|
metricsAuthUnit = "swarm-metrics-auth";
|
|
metricsAuthSocket = "/run/${metricsAuthUnit}/auth.sock";
|
|
metricsAuthCredential = id: "/run/credentials/${metricsAuthUnit}.service/${id}";
|
|
# Credential id → host path of the responder's store identity.
|
|
metricsAuthStoreFiles = {
|
|
"bao-client.pem" = deployCfg.victoriametrics.authBaoClientCertFile;
|
|
"bao-client-key.pem" = deployCfg.victoriametrics.authBaoClientKeyFile;
|
|
}
|
|
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
|
|
"bao-ca.pem" = deployCfg.bao.serverCaFile;
|
|
};
|
|
baoCfg = hyperhiveCfg.swarm.bao;
|
|
# The path Grafana's host creates the bearer at, under the KV mount. The
|
|
# same leaf as `grafanaMetricsBearerLeaf` in ./swarm-bao.nix.
|
|
grafanaBearerStorePath = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer";
|
|
in
|
|
{
|
|
# What the store IS from any hive's point of view, the name it answers on,
|
|
# is `swarm.victoriametrics` in ./swarm-victoriametrics-service.nix.
|
|
# `enable`, `package` and `retentionPeriod` are decisions of the host that
|
|
# runs it and live under `deploy.*`.
|
|
|
|
# Retention is a property of the store this host runs, not something the
|
|
# swarm has to agree on: it is read only where the container is defined,
|
|
# and a hive that is a *client* of the metrics store never consults it.
|
|
options.services.hyperhive.deploy.victoriametrics.package = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = pkgs.victoriametrics;
|
|
defaultText = lib.literalExpression "pkgs.victoriametrics";
|
|
description = "VictoriaMetrics package to run.";
|
|
};
|
|
|
|
options.services.hyperhive.deploy.victoriametrics.retentionPeriod = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "5y";
|
|
example = "90d";
|
|
description = ''
|
|
How long samples are kept.
|
|
|
|
Deliberately a high default rather than a required option: the two
|
|
failure directions are not symmetric. Too long fills a disk, which
|
|
is visible and recoverable by lowering this; too short **destroys
|
|
history**, silently and permanently. So the safe default is generous
|
|
and an operator lowers it once they have measured how fast this swarm
|
|
actually accumulates data.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.deploy.victoriametrics.authPackage = lib.mkOption {
|
|
type = lib.types.package;
|
|
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-metrics-auth";
|
|
description = ''
|
|
The `swarm-metrics-auth` package: the gateway's `auth_request` responder
|
|
that checks the bearer Grafana's metrics datasource presents on the
|
|
store's query API against the secret store.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.deploy.victoriametrics.authBaoClientCertFile = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
example = "/var/lib/swarm-bao-pki/metrics-auth.pem";
|
|
description = ''
|
|
Client certificate `swarm-metrics-auth` presents to the swarm's secret
|
|
store to read Grafana's metrics bearer. Its subject must be
|
|
{option}`services.hyperhive.deploy.bao.metricsAuthCommonName`.
|
|
|
|
Required when `deploy.victoriametrics.enable` is set; unset fails at
|
|
eval. ./glue-metrics-auth-bao-identity.nix points it at the leaf
|
|
./glue-bao-tls.nix mints on a host that runs the store; on any other
|
|
metrics-store host it is a file the operator places.
|
|
|
|
A path, never a value.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.deploy.victoriametrics.authBaoClientKeyFile = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
example = "/var/lib/swarm-bao-pki/metrics-auth-key.pem";
|
|
description = ''
|
|
Private key for {option}`services.hyperhive.deploy.victoriametrics.authBaoClientCertFile`.
|
|
A path, never a value.
|
|
'';
|
|
};
|
|
|
|
config = lib.mkIf deployCfg.victoriametrics.enable {
|
|
assertions = [
|
|
{
|
|
assertion = lib.versionAtLeast (lib.getVersion deployCfg.victoriametrics.package) "1.153.0";
|
|
message = ''
|
|
services.hyperhive.deploy.victoriametrics.package must be
|
|
VictoriaMetrics 1.153.0 or later (this one is
|
|
${lib.getVersion deployCfg.victoriametrics.package}).
|
|
|
|
The gateway reaches the store over a unix socket, and only from
|
|
1.153.0 does VictoriaMetrics create that socket under the process
|
|
umask. An older build creates it 0600 whatever the umask, so nginx
|
|
cannot connect and every read, push and scrape fails with 502.
|
|
'';
|
|
}
|
|
{
|
|
assertion =
|
|
deployCfg.victoriametrics.authBaoClientCertFile != null
|
|
&& deployCfg.victoriametrics.authBaoClientKeyFile != null;
|
|
message = ''
|
|
services.hyperhive.deploy.victoriametrics.authBaoClientCertFile and
|
|
authBaoClientKeyFile must both be set.
|
|
|
|
swarm-metrics-auth reads Grafana's metrics bearer from the swarm's
|
|
secret store with this certificate. They default to the leaf the
|
|
store mints where services.hyperhive.deploy.bao.enable is set on
|
|
this host; elsewhere, issue a leaf whose CN is
|
|
services.hyperhive.deploy.bao.metricsAuthCommonName and name it in
|
|
both options.
|
|
'';
|
|
}
|
|
];
|
|
|
|
# The gateway name, inside `deployCfg.victoriametrics.enable` — that
|
|
# guard is the load-bearing part. Every hive in a swarm may know this
|
|
# store exists, but only the host that RUNS it may claim the name; a
|
|
# client hive declaring the vhost would answer for a service it does not
|
|
# have.
|
|
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
|
|
|
# This host serves the vhost, and the hive's dnsmasq answers its name.
|
|
services.hyperhive.gateway.enable = lib.mkDefault true;
|
|
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
|
|
|
# Which check a read location's `auth_request` gets. `auth_request` takes a
|
|
# fixed URI, so it names `/__metrics_read_auth`, which rewrites to the
|
|
# target this map picks: a bearer goes to Grafana's check, anything else
|
|
# (an operator's session cookie) to authelia.
|
|
services.nginx.appendHttpConfig = ''
|
|
map $http_authorization $swarm_metrics_read_auth_target {
|
|
"~^Bearer " /__metrics_grafana_auth;
|
|
default /__metrics_authelia;
|
|
}
|
|
'';
|
|
|
|
# `removeAttrs`/`forceSSL`: authelia answers a plain-http auth subrequest
|
|
# with 400, which `auth_request` cannot read as anything but a broken
|
|
# check. Same as ./swarm-victorialogs.nix's vhost.
|
|
services.nginx.virtualHosts."${cfg.domain}" =
|
|
(builtins.removeAttrs (gatewayCfg.lib.tlsFor cfg.domain) [ "addSSL" ])
|
|
// {
|
|
forceSSL = true;
|
|
listen = gatewayCfg.lib.listen;
|
|
extraConfig = gatewayCfg.lib.securityHeaders;
|
|
locations =
|
|
readLocations
|
|
// uiLocations
|
|
// {
|
|
# Nothing reaches the store unless a location here names it. The
|
|
# store serves writes and admin calls (`/api/v1/import`,
|
|
# `/api/v1/admin/tsdb/delete_series`, ...) on the same listener as
|
|
# its queries, and authenticates none of them.
|
|
"/".return = "403";
|
|
|
|
# The swarm UI's quick link opens the bare domain. The store's own
|
|
# index page links mostly to paths refused above, so send it to vmui.
|
|
"= /".return = "302 /vmui/";
|
|
|
|
# The swarm collector's ingest route. `=` so it outranks the `/`
|
|
# prefix above.
|
|
#
|
|
# ⚠️ No `error_page 401 =302` here, and its absence is the point: a
|
|
# redirect is right for a browser and wrong for a pusher, which would
|
|
# follow it and POST its batch at a login page that answers 200 —
|
|
# ingest reporting healthy while storing nothing.
|
|
"= /opentelemetry/api/v1/push" = {
|
|
proxyPass = upstream "/opentelemetry/api/v1/push";
|
|
extraConfig = ''
|
|
auth_request /__metrics_authelia;
|
|
'';
|
|
};
|
|
|
|
# The store's own health, for the swarm collector wherever it runs:
|
|
# served on the query listener, so no exporter and no extra port. `=`
|
|
# for the same reason as the push route, and no `error_page` for the
|
|
# same reason: a scraper would parse a login page as metrics.
|
|
#
|
|
# The collector requests exactly this URL, declared as
|
|
# `publishedScrapeTargets.victoriametrics` in ./swarm-otel-service.nix;
|
|
# the two must agree, since that string is also the token's audience.
|
|
"= /metrics" = {
|
|
proxyPass = upstream "/metrics";
|
|
extraConfig = ''
|
|
auth_request /__metrics_authelia;
|
|
'';
|
|
};
|
|
|
|
# `rewrite … last` re-matches locations, so the subrequest lands in
|
|
# one of the two internal locations named by the map.
|
|
"= /__metrics_read_auth" = {
|
|
extraConfig = ''
|
|
internal;
|
|
rewrite ^ $swarm_metrics_read_auth_target last;
|
|
'';
|
|
};
|
|
|
|
# Grafana's bearer, checked by `swarm-metrics-auth`. The
|
|
# `Authorization` header rides the subrequest unchanged. No rate limit
|
|
# in front: the responder reads the store at most once per its cache
|
|
# TTL, whatever the bearers.
|
|
"= /__metrics_grafana_auth" = {
|
|
proxyPass = "http://unix:${metricsAuthSocket}:/";
|
|
extraConfig = ''
|
|
internal;
|
|
proxy_pass_request_body off;
|
|
proxy_set_header Content-Length "";
|
|
'';
|
|
};
|
|
|
|
# The authelia subrequest. For the collector's two locations above it
|
|
# admits only the collector's client; for the read and vmui locations
|
|
# it admits an operator's session (`metricsStoreRules` in
|
|
# ./swarm-authelia.nix). Same target and header set as the sibling log
|
|
# store's, which took them from `swarm-ui.nix` — `X-Original-URL` and
|
|
# `X-Original-Method` are what authelia's auth-request implementation
|
|
# reads, and the address it compares the token's audience against.
|
|
"= /__metrics_authelia" = {
|
|
proxyPass = "https://${hyperhiveCfg.swarm.authelia.domain}/api/authz/auth-request";
|
|
# nixpkgs appends its OWN `Host $host` after extraConfig, which
|
|
# would override verifiedProxyTo's — see the comment on
|
|
# verifiedProxyTo in hive-gateway/vhost-lib.nix.
|
|
recommendedProxySettings = false;
|
|
extraConfig = ''
|
|
internal;
|
|
${gatewayCfg.lib.verifiedProxyTo hyperhiveCfg.swarm.authelia.domain}
|
|
proxy_pass_request_body off;
|
|
proxy_set_header Content-Length "";
|
|
proxy_set_header X-Original-Method $request_method;
|
|
proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
|
|
proxy_set_header X-Forwarded-Proto $scheme;
|
|
proxy_set_header X-Forwarded-Host $http_host;
|
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
|
'';
|
|
};
|
|
};
|
|
};
|
|
|
|
# The socket directory, created host-side before the container starts:
|
|
# nixos-container refuses to start when a bind source is missing.
|
|
#
|
|
# Root-owned and group nginx, `0770`: the store creates its socket here
|
|
# through the group (its uid is allocated inside the container, so the
|
|
# host cannot name it), and the gateway traverses it the same way.
|
|
# Everything else is kept out.
|
|
systemd.tmpfiles.rules = [
|
|
"d ${socketDir} 0770 root ${toString nginxGid} - -"
|
|
];
|
|
|
|
# Grafana's check on the read API. On the host rather than in the store's
|
|
# container: it needs nothing of the store, only nginx's reach to its
|
|
# socket and its own store identity.
|
|
users.users.${metricsAuthUnit} = {
|
|
isSystemUser = true;
|
|
group = metricsAuthUnit;
|
|
description = "swarm metrics store auth_request responder";
|
|
};
|
|
users.groups.${metricsAuthUnit} = { };
|
|
|
|
systemd.services.${metricsAuthUnit} = {
|
|
description = "swarm metrics store auth_request responder for Grafana's bearer";
|
|
wantedBy = [ "multi-user.target" ];
|
|
# Where the store's PKI is minted on this host, the leaf is one of its
|
|
# files. Ordering only: elsewhere the unit does not exist.
|
|
after = [
|
|
"network.target"
|
|
"swarm-bao-pki.service"
|
|
];
|
|
serviceConfig = {
|
|
ExecStart = lib.concatStringsSep " " [
|
|
"${deployCfg.victoriametrics.authPackage}/bin/swarm-metrics-auth"
|
|
"--socket ${metricsAuthSocket}"
|
|
# The role ./swarm-bao.nix writes on the store's `cert` mount.
|
|
"--store-cert-role swarm-metrics-auth"
|
|
"--store-path ${grafanaBearerStorePath}"
|
|
];
|
|
# Paths only: `argv` is world-readable via /proc/<pid>/cmdline.
|
|
LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") metricsAuthStoreFiles;
|
|
# ⚠️ A static user, never `DynamicUser`: with `RuntimeDirectoryPreserve`
|
|
# systemd puts a dynamic user's runtime directory under
|
|
# `/run/private/` (0700 root:root), which nginx cannot traverse.
|
|
User = metricsAuthUnit;
|
|
Group = metricsAuthUnit;
|
|
RuntimeDirectory = metricsAuthUnit;
|
|
# Traverse-only for others, so nginx reaches the socket path.
|
|
RuntimeDirectoryMode = "0751";
|
|
# The path never vanishes from under a running nginx; the responder
|
|
# unlinks a stale socket as it starts.
|
|
RuntimeDirectoryPreserve = "yes";
|
|
Restart = "on-failure";
|
|
RestartSec = "5s";
|
|
SyslogIdentifier = metricsAuthUnit;
|
|
ProtectSystem = "strict";
|
|
ProtectHome = true;
|
|
PrivateTmp = "disconnected";
|
|
PrivateDevices = true;
|
|
NoNewPrivileges = true;
|
|
RestrictSUIDSGID = true;
|
|
RemoveIPC = true;
|
|
};
|
|
environment = {
|
|
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
|
BAO_CLIENT_CERT = metricsAuthCredential "bao-client.pem";
|
|
BAO_CLIENT_KEY = metricsAuthCredential "bao-client-key.pem";
|
|
}
|
|
// lib.optionalAttrs (metricsAuthStoreFiles ? "bao-ca.pem") {
|
|
BAO_CACERT = metricsAuthCredential "bao-ca.pem";
|
|
};
|
|
};
|
|
|
|
containers.${cfg.machine} = {
|
|
autoStart = true;
|
|
ephemeral = false;
|
|
# Journal files on the host, not inside the container: nixpkgs hardcodes
|
|
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
|
|
extraFlags = [ "--link-journal=host" ];
|
|
inherit privateNetwork;
|
|
|
|
# A bind mount rather than a path both sides happen to know: a
|
|
# container's `/run` is an nspawn tmpfs the host cannot see into.
|
|
bindMounts.${socketDir} = {
|
|
hostPath = socketDir;
|
|
isReadOnly = false;
|
|
};
|
|
|
|
config =
|
|
{ config, pkgs, ... }:
|
|
{
|
|
imports = [ ../container-modules/swarm-container.nix ];
|
|
|
|
# No resolver unit: with no interface but loopback there is no
|
|
# nameserver to reach, and the store resolves no names.
|
|
services.hyperhive.swarmContainer = {
|
|
inherit privateNetwork;
|
|
writesOwnResolvConf = false;
|
|
};
|
|
|
|
# The socket's group is the store's primary group, so that group is
|
|
# nginx's gid. A static user, not nixpkgs' DynamicUser: a dynamic
|
|
# user's group is allocated per start, and nginx is never in it.
|
|
users.groups.gateway-nginx.gid = nginxGid;
|
|
users.users.victoriametrics = {
|
|
isSystemUser = true;
|
|
group = "gateway-nginx";
|
|
};
|
|
|
|
systemd.services.victoriametrics = {
|
|
serviceConfig = {
|
|
DynamicUser = lib.mkForce false;
|
|
User = "victoriametrics";
|
|
Group = "gateway-nginx";
|
|
# The store creates its socket under the process umask, so
|
|
# `0007` makes it `0770` and nginx's group can connect.
|
|
UMask = "0007";
|
|
};
|
|
# nixpkgs' readiness loop curls `http://<listenAddress>/ping`,
|
|
# which no `unix:` address answers.
|
|
postStart = lib.mkForce ''
|
|
until ${lib.getExe pkgs.curl} -s -o /dev/null --unix-socket ${socketPath} http://localhost/ping; do
|
|
sleep 1;
|
|
done
|
|
'';
|
|
};
|
|
|
|
# Moves the store's history out of the DynamicUser layout, where
|
|
# `/var/lib/<stateDir>` is a symlink into `/var/lib/private/`, to the
|
|
# plain directory the static user's StateDirectory is. Runs before
|
|
# the store on every start and does nothing once
|
|
# `/var/lib/private/<stateDir>` is gone. When both paths hold data
|
|
# it moves neither and the store starts on the plain one.
|
|
systemd.services.victoriametrics-state-migrate =
|
|
let
|
|
old = "/var/lib/private/${config.services.victoriametrics.stateDir}";
|
|
new = "/var/lib/${config.services.victoriametrics.stateDir}";
|
|
in
|
|
{
|
|
description = "move VictoriaMetrics' DynamicUser state directory to its static path";
|
|
before = [ "victoriametrics.service" ];
|
|
requiredBy = [ "victoriametrics.service" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
SyslogIdentifier = "victoriametrics-state-migrate";
|
|
};
|
|
path = [ pkgs.coreutils ];
|
|
script = ''
|
|
set -eu
|
|
old=${lib.escapeShellArg old}
|
|
new=${lib.escapeShellArg new}
|
|
if [ ! -d "$old" ] || [ -L "$old" ]; then
|
|
exit 0
|
|
fi
|
|
if [ -L "$new" ]; then
|
|
rm "$new"
|
|
elif [ -d "$new" ]; then
|
|
if [ -n "$(ls -A "$new")" ]; then
|
|
echo "victoriametrics-state-migrate: $old and $new both hold data; moving neither" >&2
|
|
exit 0
|
|
fi
|
|
rmdir "$new"
|
|
fi
|
|
mv "$old" "$new"
|
|
chown -R victoriametrics:gateway-nginx "$new"
|
|
echo "victoriametrics-state-migrate: moved $old to $new"
|
|
'';
|
|
};
|
|
|
|
services.victoriametrics = {
|
|
enable = true;
|
|
package = deployCfg.victoriametrics.package;
|
|
retentionPeriod = deployCfg.victoriametrics.retentionPeriod;
|
|
|
|
# ⚠️ A socket, not upstream's default `:8428` on every interface:
|
|
# the gateway is the only intended client, and this listener
|
|
# carries an unauthenticated write endpoint (see the OTLP note
|
|
# below).
|
|
listenAddress = "unix:${socketPath}";
|
|
};
|
|
};
|
|
};
|
|
};
|
|
|
|
# 🔑 OTLP ingest needs no flag. Measured against the pinned 1.146.0 rather
|
|
# than inferred from the module's option list, which has no OTLP switch and
|
|
# so reads as though the feature were absent: the running server answers
|
|
# `POST /opentelemetry/api/v1/push` with 200 (a nonexistent path answers
|
|
# 400, so that 200 means the route exists). The `-opentelemetry.*` flags
|
|
# only tune naming and limits, and are reachable via `extraOptions` if a
|
|
# deployment ever needs them.
|
|
#
|
|
# ⚠️ That endpoint is unauthenticated, which is why `listenAddress` above is
|
|
# a socket only the gateway can reach: the only route to it is the vhost's
|
|
# ingest location, which authenticates. The collector is still the only writer, but
|
|
# it now arrives by the swarm name rather than over loopback, because it need
|
|
# not share a host with this store.
|
|
}
|