e92f118bda
swarm-metrics reads the swarm's VictoriaMetrics from inside an agent
container: query, query-range, series, labels and label-values, each one
GET against the store's Prometheus read API through the gateway, printing
the store's JSON. It authenticates the way swarm-logs does, with the
agent's own queue secret read from bao under the agent's leaf, sent as
a swarm-agent bearer.
On the metrics store's host the read API's auth map gains an agent arm
ahead of the Grafana one. It routes to swarm-metrics-agent-auth, a
swarm-logs-auth instance logged in under the existing swarm-metrics-auth
leaf, whose bao grant now includes swarm/agents/+/queue read. The check is
rate-limited per address like the log store's
(deploy.victoriametrics.agentAuth{Rate,Burst,VerdictTtl}).
nix/agent-modules/metrics.nix installs the wrapped binary when the hive
forwards HIVE_AGENT_METRICS_URL (from swarm.victoriametrics.domain) and a
store address, the same gate as swarm-logs. Adds the base:swarm-metrics
skill and the CLI reference with its freshness check.
Fixes #5395
886 lines
38 KiB
Nix
886 lines
38 KiB
Nix
# The swarm's metrics store: one VictoriaMetrics for the whole swarm, in a
|
|
# `swarm-victoriametrics` nixos-container.
|
|
#
|
|
# A LOCAL time-series database rather than only an external sink, and that is
|
|
# the point rather than a convenience: the swarm dashboard has to stay
|
|
# readable when the outside world is unreachable. Same failure-domain property
|
|
# the hive-status KV has — a view of the system must not depend on the system
|
|
# it is viewing being healthy.
|
|
#
|
|
# It is the collector that feeds this (the gateway OTEL collector), not the
|
|
# agents directly: one ingest point per swarm, authenticated there.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.swarm.victoriametrics;
|
|
deployCfg = config.services.hyperhive.deploy;
|
|
hyperhiveCfg = config.services.hyperhive;
|
|
gatewayCfg = hyperhiveCfg.gateway;
|
|
|
|
# A netns of its own with no interface but loopback: the store makes no
|
|
# outbound calls, the gateway reaches it over `socketPath` below, and the
|
|
# journal forwarder leaves only through `egressSocket`, so nothing on the host
|
|
# or the hive bridge can dial its listener.
|
|
privateNetwork = true;
|
|
|
|
# Fixed, not an option: the directory's mode is what keeps the socket
|
|
# private, so it holds this one socket and nothing else.
|
|
socketDir = "/run/swarm-victoriametrics";
|
|
socketPath = "${socketDir}/victoriametrics.sock";
|
|
upstream = path: "http://unix:${socketPath}:${path}";
|
|
|
|
# The gid the gateway's nginx runs as. Only the number crosses into the
|
|
# container, which knows it as `gateway-nginx`.
|
|
nginxGid = config.ids.gids.nginx;
|
|
|
|
# The store's read API, which the vhost forwards to a caller holding
|
|
# Grafana's bearer, an agent's own bearer, or an operator's session. Each
|
|
# location's `proxy_pass` carries the path, so the store receives the path
|
|
# nginx matched, not the caller's unnormalised spelling.
|
|
#
|
|
# Grafana's Prometheus datasource reads the query, series, label, metadata,
|
|
# buildinfo, exemplar, rules and alerts routes; the store answers the last
|
|
# three with fixed placeholders. vmui (`uiLocations` below) is static files
|
|
# plus the query API and the remaining read-only routes here. Paths read off
|
|
# the pinned build's `app/vmselect/main.go` and vmui's `src/`.
|
|
readPaths = [
|
|
"/api/v1/query"
|
|
"/api/v1/query_range"
|
|
"/api/v1/query_exemplars"
|
|
"/api/v1/series"
|
|
"/api/v1/labels"
|
|
"/api/v1/metadata"
|
|
"/api/v1/rules"
|
|
"/api/v1/alerts"
|
|
"/api/v1/status/buildinfo"
|
|
"/api/v1/export"
|
|
"/api/v1/export/csv"
|
|
"/api/v1/status/tsdb"
|
|
"/api/v1/status/metric_names_stats"
|
|
"/api/v1/status/top_queries"
|
|
"/api/v1/status/active_queries"
|
|
"/expand-with-exprs"
|
|
"/prettify-query"
|
|
"/metric-relabel-debug"
|
|
];
|
|
# The store serves only `<name>/values` under `/api/v1/label/`.
|
|
readPrefixes = [ "/api/v1/label/" ];
|
|
|
|
# Bare `auth_request`, no login redirect: a machine caller handed
|
|
# `error_page 401 =302` follows it and reads authelia's login page as a 200
|
|
# holding no series. `/__metrics_read_auth` picks the check from the
|
|
# caller's `Authorization` (`appendHttpConfig` below).
|
|
readAuth = ''
|
|
auth_request /__metrics_read_auth;
|
|
'';
|
|
readLocations =
|
|
let
|
|
to = path: {
|
|
proxyPass = upstream path;
|
|
extraConfig = readAuth;
|
|
};
|
|
in
|
|
lib.listToAttrs (
|
|
map (p: lib.nameValuePair "= ${p}" (to p)) readPaths
|
|
++ map (p: lib.nameValuePair "^~ ${p}" (to p)) readPrefixes
|
|
);
|
|
|
|
# The same three lines as `swarmAuthRequest` in ./swarm-victorialogs.nix,
|
|
# for the same browser: an operator without a session is sent to log in.
|
|
swarmAuthRequest = ''
|
|
auth_request /__metrics_authelia;
|
|
auth_request_set $target_url $scheme://$http_host$request_uri;
|
|
error_page 401 =302 https://${hyperhiveCfg.swarm.authelia.domain}/?rd=$target_url;
|
|
'';
|
|
# vmui's static files and UI config. Only a browser loads these, and its
|
|
# queries then go to the read locations above with the same session cookie.
|
|
uiLocations = {
|
|
"= /vmui" = {
|
|
proxyPass = upstream "/vmui";
|
|
extraConfig = swarmAuthRequest;
|
|
};
|
|
"^~ /vmui/" = {
|
|
proxyPass = upstream "/vmui/";
|
|
extraConfig = swarmAuthRequest;
|
|
};
|
|
};
|
|
|
|
# The `auth_request` responder for Grafana's bearer (`swarm-metrics-auth`).
|
|
# Its socket's directory holds that socket alone and is traverse-only for
|
|
# others, the shape `swarm-logs-auth` in ./swarm-victorialogs.nix has.
|
|
metricsAuthUnit = "swarm-metrics-auth";
|
|
metricsAuthSocket = "/run/${metricsAuthUnit}/auth.sock";
|
|
metricsAuthCredential = id: "/run/credentials/${metricsAuthUnit}.service/${id}";
|
|
# Credential id → host path of the responder's store identity.
|
|
metricsAuthStoreFiles = {
|
|
"bao-client.pem" = deployCfg.victoriametrics.authBaoClientCertFile;
|
|
"bao-client-key.pem" = deployCfg.victoriametrics.authBaoClientKeyFile;
|
|
}
|
|
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
|
|
"bao-ca.pem" = deployCfg.bao.serverCaFile;
|
|
};
|
|
baoCfg = hyperhiveCfg.swarm.bao;
|
|
# The path Grafana's host creates the bearer at, under the KV mount. The
|
|
# same leaf as `grafanaMetricsBearerLeaf` in ./swarm-bao.nix.
|
|
grafanaBearerStorePath = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer";
|
|
|
|
# The agents' check on the read API: a `swarm-logs-auth` binary, logging in
|
|
# under this host's `swarm-metrics-auth` leaf, whose grant covers every
|
|
# agent's queue secret. The log store's host runs `swarm-metrics-auth` under
|
|
# its `swarm-logs-auth` leaf the same way.
|
|
agentAuthUnit = "swarm-metrics-agent-auth";
|
|
agentAuthSocket = "/run/${agentAuthUnit}/auth.sock";
|
|
agentAuthCredential = id: "/run/credentials/${agentAuthUnit}.service/${id}";
|
|
|
|
# ── the container's journal forwarder ───────────────────────────────
|
|
#
|
|
# With no interface but loopback, the forwarder reaches nothing by name. A
|
|
# relay in the container listens on `relayPort` and forwards each connection
|
|
# to `egressSocket`, a socket this host's nginx listens on and binds into
|
|
# the container. nginx sends `/<machine>/` to the swarm collector's route
|
|
# (`serviceProducers.swarm-victoriametrics` in ./swarm-otel-service.nix) and
|
|
# the token endpoint to authelia, both by name and verified. The collector
|
|
# library dials no unix socket itself, so the relay is needed.
|
|
#
|
|
# The client secret is fetched on the host, under the responder's store
|
|
# identity, and bound in read-only: the container cannot reach the store.
|
|
forwarderClientId = cfg.otel.clientId;
|
|
gatewayPortSuffix = lib.optionalString (
|
|
gatewayCfg.httpsPort != 443
|
|
) ":${toString gatewayCfg.httpsPort}";
|
|
otelDomain = hyperhiveCfg.swarm.otel.domain;
|
|
autheliaDomain = hyperhiveCfg.swarm.authelia.domain;
|
|
|
|
# Its own directory, not `socketDir`: that one's mode admits nginx to the
|
|
# store's listener, and this one admits the container to nginx's.
|
|
egressDir = "/run/swarm-victoriametrics-egress";
|
|
egressSocket = "${egressDir}/egress.sock";
|
|
relayUnit = "swarm-victoriametrics-egress-relay";
|
|
# Loopback in the container's own netns.
|
|
relayPort = 4380;
|
|
relayUrl = "http://127.0.0.1:${toString relayPort}";
|
|
forwarderTelemetryPort = 8900;
|
|
|
|
# The forwarder's way out, see `egressSocket`. Every other path is refused,
|
|
# so the container reaches the two routes its forwarder uses and nothing
|
|
# else on the network.
|
|
egressServer = ''
|
|
server {
|
|
listen unix:${egressSocket};
|
|
|
|
location / {
|
|
return 403;
|
|
}
|
|
|
|
location ^~ /${cfg.machine}/ {
|
|
proxy_pass https://${otelDomain}${gatewayPortSuffix};
|
|
${gatewayCfg.lib.verifiedProxyTo otelDomain}
|
|
}
|
|
|
|
location = /api/oidc/token {
|
|
proxy_pass https://${autheliaDomain};
|
|
${gatewayCfg.lib.verifiedProxyTo autheliaDomain}
|
|
}
|
|
}
|
|
'';
|
|
|
|
forwarderFetchUnit = "swarm-victoriametrics-forwarder-oidc";
|
|
forwarderStoreSecretPath = "secret/swarm/services/${forwarderClientId}/oidc/client";
|
|
# On the host's /run (tmpfs), and seen read-only in the container at the
|
|
# same path. The file is not created ahead of the fetch: `LoadCredential`
|
|
# refuses a missing source, so the forwarder stays down until the store
|
|
# answers and then resumes from its saved read cursor.
|
|
forwarderSecretDir = "/run/${forwarderFetchUnit}";
|
|
forwarderSecret = "${forwarderSecretDir}/${forwarderClientId}.secret";
|
|
|
|
storeRetry = import ./lib/store-retry.nix { };
|
|
storeRefreshTimer = import ./lib/store-refresh-timer.nix { };
|
|
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
|
|
refreshConsumer = import ./lib/refresh-consumer.nix { };
|
|
in
|
|
{
|
|
# What the store IS from any hive's point of view, the name it answers on,
|
|
# is `swarm.victoriametrics` in ./swarm-victoriametrics-service.nix.
|
|
# `enable`, `package` and `retentionPeriod` are decisions of the host that
|
|
# runs it and live under `deploy.*`.
|
|
|
|
# Retention is a property of the store this host runs, not something the
|
|
# swarm has to agree on: it is read only where the container is defined,
|
|
# and a hive that is a *client* of the metrics store never consults it.
|
|
options.services.hyperhive.deploy.victoriametrics.package = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = pkgs.victoriametrics;
|
|
defaultText = lib.literalExpression "pkgs.victoriametrics";
|
|
description = "VictoriaMetrics package to run.";
|
|
};
|
|
|
|
options.services.hyperhive.deploy.victoriametrics.retentionPeriod = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "5y";
|
|
example = "90d";
|
|
description = ''
|
|
How long samples are kept.
|
|
|
|
Deliberately a high default rather than a required option: the two
|
|
failure directions are not symmetric. Too long fills a disk, which
|
|
is visible and recoverable by lowering this; too short **destroys
|
|
history**, silently and permanently. So the safe default is generous
|
|
and an operator lowers it once they have measured how fast this swarm
|
|
actually accumulates data.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.deploy.victoriametrics.authPackage = lib.mkOption {
|
|
type = lib.types.package;
|
|
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-metrics-auth";
|
|
description = ''
|
|
The `swarm-metrics-auth` package: the gateway's `auth_request` responder
|
|
that checks the bearer Grafana's metrics datasource presents on the
|
|
store's query API against the secret store.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.deploy.victoriametrics.authBaoClientCertFile = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
example = "/var/lib/swarm-bao-pki/metrics-auth.pem";
|
|
description = ''
|
|
Client certificate `swarm-metrics-auth` presents to the swarm's secret
|
|
store to read Grafana's metrics bearer, and `swarm-metrics-agent-auth`
|
|
to read agents' queue secrets. Its subject must be
|
|
{option}`services.hyperhive.deploy.bao.metricsAuthCommonName`.
|
|
|
|
Required when `deploy.victoriametrics.enable` is set; unset fails at
|
|
eval. ./glue-metrics-auth-bao-identity.nix points it at the leaf
|
|
./glue-bao-tls.nix mints on a host that runs the store; on any other
|
|
metrics-store host it is a file the operator places.
|
|
|
|
A path, never a value.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.deploy.victoriametrics.authBaoClientKeyFile = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
example = "/var/lib/swarm-bao-pki/metrics-auth-key.pem";
|
|
description = ''
|
|
Private key for {option}`services.hyperhive.deploy.victoriametrics.authBaoClientCertFile`.
|
|
A path, never a value.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.deploy.victoriametrics.agentAuthVerdictTtl = lib.mkOption {
|
|
type = lib.types.ints.unsigned;
|
|
default = 60;
|
|
description = ''
|
|
Seconds `swarm-metrics-agent-auth` grants an agent bearer it has
|
|
verified again without reading the secret store. A denial is never
|
|
remembered.
|
|
|
|
This is also how long a re-minted or deleted queue secret keeps working
|
|
at the metrics store's read API after its last successful check. `0`
|
|
checks the store on every query.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.deploy.victoriametrics.agentAuthRate = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "50r/s";
|
|
example = "30r/m";
|
|
description = ''
|
|
nginx `limit_req` rate, per client address, of read API requests
|
|
carrying a `swarm-agent.` bearer, which nginx checks with
|
|
`swarm-metrics-agent-auth`. Each such request that the responder has
|
|
not granted within
|
|
{option}`services.hyperhive.deploy.victoriametrics.agentAuthVerdictTtl`
|
|
costs a secret store login and read, whether the bearer is right or not.
|
|
|
|
Keyed by client address, so every agent on one host shares one budget.
|
|
Requests over the rate and its burst are answered 403.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.deploy.victoriametrics.agentAuthBurst = lib.mkOption {
|
|
type = lib.types.ints.positive;
|
|
default = 500;
|
|
description = ''
|
|
Requests a client address may make above
|
|
{option}`services.hyperhive.deploy.victoriametrics.agentAuthRate` before
|
|
nginx answers 403, served without delay.
|
|
'';
|
|
};
|
|
|
|
config = lib.mkIf deployCfg.victoriametrics.enable {
|
|
assertions = [
|
|
{
|
|
assertion = lib.versionAtLeast (lib.getVersion deployCfg.victoriametrics.package) "1.153.0";
|
|
message = ''
|
|
services.hyperhive.deploy.victoriametrics.package must be
|
|
VictoriaMetrics 1.153.0 or later (this one is
|
|
${lib.getVersion deployCfg.victoriametrics.package}).
|
|
|
|
The gateway reaches the store over a unix socket, and only from
|
|
1.153.0 does VictoriaMetrics create that socket under the process
|
|
umask. An older build creates it 0600 whatever the umask, so nginx
|
|
cannot connect and every read, push and scrape fails with 502.
|
|
'';
|
|
}
|
|
{
|
|
assertion =
|
|
deployCfg.victoriametrics.authBaoClientCertFile != null
|
|
&& deployCfg.victoriametrics.authBaoClientKeyFile != null;
|
|
message = ''
|
|
services.hyperhive.deploy.victoriametrics.authBaoClientCertFile and
|
|
authBaoClientKeyFile must both be set.
|
|
|
|
swarm-metrics-auth reads Grafana's metrics bearer, and
|
|
swarm-metrics-agent-auth agents' queue secrets, from the swarm's
|
|
secret store with this certificate. They default to the leaf the
|
|
store mints where services.hyperhive.deploy.bao.enable is set on
|
|
this host; elsewhere, issue a leaf whose CN is
|
|
services.hyperhive.deploy.bao.metricsAuthCommonName and name it in
|
|
both options.
|
|
'';
|
|
}
|
|
{
|
|
assertion =
|
|
(hyperhiveCfg.swarm.otel.serviceProducers.${cfg.machine}.clientId or null) == forwarderClientId;
|
|
message = ''
|
|
services.hyperhive.swarm.otel.serviceProducers has no entry
|
|
"${cfg.machine}" with clientId "${forwarderClientId}", so the swarm
|
|
collector has no receiver for the metrics store's journal forwarder.
|
|
Declare it in swarm-otel-service.nix.
|
|
'';
|
|
}
|
|
];
|
|
|
|
# The gateway name, inside `deployCfg.victoriametrics.enable` — that
|
|
# guard is the load-bearing part. Every hive in a swarm may know this
|
|
# store exists, but only the host that RUNS it may claim the name; a
|
|
# client hive declaring the vhost would answer for a service it does not
|
|
# have.
|
|
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
|
|
|
# This host serves the vhost, and the hive's dnsmasq answers its name.
|
|
services.hyperhive.gateway.enable = lib.mkDefault true;
|
|
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
|
|
|
# Which check a read location's `auth_request` gets. `auth_request` takes a
|
|
# fixed URI, so it names `/__metrics_read_auth`, which rewrites to the
|
|
# target this map picks: an agent's `swarm-agent.` bearer goes to the
|
|
# agents' check, any other bearer to Grafana's, anything else (an
|
|
# operator's session cookie) to authelia. A map tries its regexes in
|
|
# order, so the agent arm comes first. After the forwarder's egress
|
|
# listener, `egressServer`.
|
|
services.nginx.appendHttpConfig = egressServer + ''
|
|
map $http_authorization $swarm_metrics_read_auth_target {
|
|
"~^Bearer swarm-agent\." /__metrics_agent_auth;
|
|
"~^Bearer " /__metrics_grafana_auth;
|
|
default /__metrics_authelia;
|
|
}
|
|
limit_req_zone $binary_remote_addr zone=swarm_metrics_agent_auth:1m rate=${deployCfg.victoriametrics.agentAuthRate};
|
|
'';
|
|
|
|
# `forceSSL`: authelia answers a plain-http auth subrequest
|
|
# with 400, which `auth_request` cannot read as anything but a broken
|
|
# check. Same as ./swarm-victorialogs.nix's vhost.
|
|
services.hyperhive.gateway.vhosts."${cfg.domain}" = {
|
|
forceSSL = true;
|
|
locations =
|
|
readLocations
|
|
// uiLocations
|
|
// {
|
|
# Nothing reaches the store unless a location here names it. The
|
|
# store serves writes and admin calls (`/api/v1/import`,
|
|
# `/api/v1/admin/tsdb/delete_series`, ...) on the same listener as
|
|
# its queries, and authenticates none of them.
|
|
"/".return = "403";
|
|
|
|
# The swarm UI's quick link opens the bare domain. The store's own
|
|
# index page links mostly to paths refused above, so send it to vmui.
|
|
"= /".return = "302 /vmui/";
|
|
|
|
# The swarm collector's ingest route. `=` so it outranks the `/`
|
|
# prefix above.
|
|
#
|
|
# ⚠️ No `error_page 401 =302` here, and its absence is the point: a
|
|
# redirect is right for a browser and wrong for a pusher, which would
|
|
# follow it and POST its batch at a login page that answers 200 —
|
|
# ingest reporting healthy while storing nothing.
|
|
"= /opentelemetry/api/v1/push" = {
|
|
proxyPass = upstream "/opentelemetry/api/v1/push";
|
|
extraConfig = ''
|
|
auth_request /__metrics_authelia;
|
|
'';
|
|
};
|
|
|
|
# The store's own health, for the swarm collector wherever it runs:
|
|
# served on the query listener, so no exporter and no extra port. `=`
|
|
# for the same reason as the push route, and no `error_page` for the
|
|
# same reason: a scraper would parse a login page as metrics.
|
|
#
|
|
# The collector requests exactly this URL, declared as
|
|
# `publishedScrapeTargets.victoriametrics` in ./swarm-otel-service.nix;
|
|
# the two must agree, since that string is also the token's audience.
|
|
"= /metrics" = {
|
|
proxyPass = upstream "/metrics";
|
|
extraConfig = ''
|
|
auth_request /__metrics_authelia;
|
|
'';
|
|
};
|
|
|
|
# `rewrite … last` re-matches locations, so the subrequest lands in
|
|
# one of the two internal locations named by the map.
|
|
"= /__metrics_read_auth" = {
|
|
extraConfig = ''
|
|
internal;
|
|
rewrite ^ $swarm_metrics_read_auth_target last;
|
|
'';
|
|
};
|
|
|
|
# Grafana's bearer, checked by `swarm-metrics-auth`. The
|
|
# `Authorization` header rides the subrequest unchanged. No rate limit
|
|
# in front: the responder reads the store at most once per its cache
|
|
# TTL, whatever the bearers.
|
|
"= /__metrics_grafana_auth" = {
|
|
proxyPass = "http://unix:${metricsAuthSocket}:/";
|
|
extraConfig = ''
|
|
internal;
|
|
proxy_pass_request_body off;
|
|
proxy_set_header Content-Length "";
|
|
'';
|
|
};
|
|
|
|
# An agent's bearer, checked by `swarm-metrics-agent-auth`. Any
|
|
# well-formed agent bearer the responder has not recently granted
|
|
# costs a store login and read, so the rate per client address is
|
|
# capped here. 403 when over: `auth_request` passes only 401 and 403
|
|
# through, and `swarm-metrics` retries a 401.
|
|
"= /__metrics_agent_auth" = {
|
|
proxyPass = "http://unix:${agentAuthSocket}:/";
|
|
extraConfig = ''
|
|
internal;
|
|
limit_req zone=swarm_metrics_agent_auth burst=${toString deployCfg.victoriametrics.agentAuthBurst} nodelay;
|
|
limit_req_status 403;
|
|
proxy_pass_request_body off;
|
|
proxy_set_header Content-Length "";
|
|
'';
|
|
};
|
|
|
|
# The authelia subrequest. For the collector's two locations above it
|
|
# admits only the collector's client; for the read and vmui locations
|
|
# it admits an operator's session (`metricsStoreRules` in
|
|
# ./swarm-authelia.nix). Same target and header set as the sibling log
|
|
# store's, which took them from `swarm-ui.nix` — `X-Original-URL` and
|
|
# `X-Original-Method` are what authelia's auth-request implementation
|
|
# reads, and the address it compares the token's audience against.
|
|
"= /__metrics_authelia" = {
|
|
proxyPass = "https://${hyperhiveCfg.swarm.authelia.domain}/api/authz/auth-request";
|
|
# nixpkgs appends its OWN `Host $host` after extraConfig, which
|
|
# would override verifiedProxyTo's — see the comment on
|
|
# verifiedProxyTo in hive-gateway/vhost-lib.nix.
|
|
recommendedProxySettings = false;
|
|
extraConfig = ''
|
|
internal;
|
|
${gatewayCfg.lib.verifiedProxyTo hyperhiveCfg.swarm.authelia.domain}
|
|
proxy_pass_request_body off;
|
|
proxy_set_header Content-Length "";
|
|
proxy_set_header X-Original-Method $request_method;
|
|
proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
|
|
proxy_set_header X-Forwarded-Proto $scheme;
|
|
proxy_set_header X-Forwarded-Host $http_host;
|
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
|
'';
|
|
};
|
|
};
|
|
};
|
|
|
|
# The socket directory, created host-side before the container starts:
|
|
# nixos-container refuses to start when a bind source is missing.
|
|
#
|
|
# Root-owned and group nginx, `0770`: the store creates its socket here
|
|
# through the group (its uid is allocated inside the container, so the
|
|
# host cannot name it), and the gateway traverses it the same way.
|
|
# Everything else is kept out.
|
|
systemd.tmpfiles.rules = [
|
|
"d ${socketDir} 0770 root ${toString nginxGid} - -"
|
|
# The relay in the container connects as root; nothing else on the host
|
|
# but nginx may.
|
|
"d ${egressDir} 0750 ${config.services.nginx.user} ${config.services.nginx.group} - -"
|
|
# The forwarder secret's bind source, so it exists when nspawn sets the
|
|
# mount up. Not the file: see `forwarderSecretDir`.
|
|
"d ${forwarderSecretDir} 0700 root root -"
|
|
];
|
|
|
|
# Grafana's check on the read API. On the host rather than in the store's
|
|
# container: it needs nothing of the store, only nginx's reach to its
|
|
# socket and its own store identity.
|
|
users.users.${metricsAuthUnit} = {
|
|
isSystemUser = true;
|
|
group = metricsAuthUnit;
|
|
description = "swarm metrics store auth_request responder";
|
|
};
|
|
users.groups.${metricsAuthUnit} = { };
|
|
|
|
systemd.services.${metricsAuthUnit} = {
|
|
description = "swarm metrics store auth_request responder for Grafana's bearer";
|
|
wantedBy = [ "multi-user.target" ];
|
|
# Where the store's PKI is minted on this host, the leaf is one of its
|
|
# files. Ordering only: elsewhere the unit does not exist.
|
|
after = [
|
|
"network.target"
|
|
"swarm-bao-pki.service"
|
|
];
|
|
serviceConfig = {
|
|
ExecStart = lib.concatStringsSep " " [
|
|
"${deployCfg.victoriametrics.authPackage}/bin/swarm-metrics-auth"
|
|
"--socket ${metricsAuthSocket}"
|
|
# The role ./swarm-bao.nix writes on the store's `cert` mount.
|
|
"--store-cert-role swarm-metrics-auth"
|
|
"--store-path ${grafanaBearerStorePath}"
|
|
];
|
|
# Paths only: `argv` is world-readable via /proc/<pid>/cmdline.
|
|
LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") metricsAuthStoreFiles;
|
|
# ⚠️ A static user, never `DynamicUser`: with `RuntimeDirectoryPreserve`
|
|
# systemd puts a dynamic user's runtime directory under
|
|
# `/run/private/` (0700 root:root), which nginx cannot traverse.
|
|
User = metricsAuthUnit;
|
|
Group = metricsAuthUnit;
|
|
RuntimeDirectory = metricsAuthUnit;
|
|
# Traverse-only for others, so nginx reaches the socket path.
|
|
RuntimeDirectoryMode = "0751";
|
|
# The path never vanishes from under a running nginx; the responder
|
|
# unlinks a stale socket as it starts.
|
|
RuntimeDirectoryPreserve = "yes";
|
|
Restart = "on-failure";
|
|
RestartSec = "5s";
|
|
SyslogIdentifier = metricsAuthUnit;
|
|
ProtectSystem = "strict";
|
|
ProtectHome = true;
|
|
PrivateTmp = "disconnected";
|
|
PrivateDevices = true;
|
|
NoNewPrivileges = true;
|
|
RestrictSUIDSGID = true;
|
|
RemoveIPC = true;
|
|
};
|
|
environment = {
|
|
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
|
BAO_CLIENT_CERT = metricsAuthCredential "bao-client.pem";
|
|
BAO_CLIENT_KEY = metricsAuthCredential "bao-client-key.pem";
|
|
}
|
|
// lib.optionalAttrs (metricsAuthStoreFiles ? "bao-ca.pem") {
|
|
BAO_CACERT = metricsAuthCredential "bao-ca.pem";
|
|
};
|
|
};
|
|
|
|
# The agents' check on the read API, see `agentAuthUnit`.
|
|
users.users.${agentAuthUnit} = {
|
|
isSystemUser = true;
|
|
group = agentAuthUnit;
|
|
description = "swarm metrics store auth_request responder for agent credentials";
|
|
};
|
|
users.groups.${agentAuthUnit} = { };
|
|
|
|
systemd.services.${agentAuthUnit} = {
|
|
description = "swarm metrics store auth_request responder for agent credentials";
|
|
wantedBy = [ "multi-user.target" ];
|
|
after = [
|
|
"network.target"
|
|
"swarm-bao-pki.service"
|
|
];
|
|
serviceConfig = {
|
|
ExecStart = lib.concatStringsSep " " [
|
|
"${deployCfg.victorialogs.authPackage}/bin/swarm-logs-auth"
|
|
"--socket ${agentAuthSocket}"
|
|
"--store-cert-role swarm-metrics-auth"
|
|
"--verdict-ttl ${toString deployCfg.victoriametrics.agentAuthVerdictTtl}"
|
|
];
|
|
LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") metricsAuthStoreFiles;
|
|
# ⚠️ A static user, never `DynamicUser`: see the `swarm-metrics-auth` unit above.
|
|
User = agentAuthUnit;
|
|
Group = agentAuthUnit;
|
|
RuntimeDirectory = agentAuthUnit;
|
|
RuntimeDirectoryMode = "0751";
|
|
RuntimeDirectoryPreserve = "yes";
|
|
Restart = "on-failure";
|
|
RestartSec = "5s";
|
|
SyslogIdentifier = agentAuthUnit;
|
|
ProtectSystem = "strict";
|
|
ProtectHome = true;
|
|
PrivateTmp = "disconnected";
|
|
PrivateDevices = true;
|
|
NoNewPrivileges = true;
|
|
RestrictSUIDSGID = true;
|
|
RemoveIPC = true;
|
|
};
|
|
environment = {
|
|
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
|
BAO_CLIENT_CERT = agentAuthCredential "bao-client.pem";
|
|
BAO_CLIENT_KEY = agentAuthCredential "bao-client-key.pem";
|
|
}
|
|
// lib.optionalAttrs (metricsAuthStoreFiles ? "bao-ca.pem") {
|
|
BAO_CACERT = agentAuthCredential "bao-ca.pem";
|
|
};
|
|
};
|
|
|
|
# nginx runs as its own user under `ProtectSystem=strict`, so it can only
|
|
# create the socket where it may write. It removes the socket on a clean
|
|
# exit only; a stale one after a crash fails every later `bind`, and with
|
|
# it the whole gateway, so it goes before each start. `-`: the directory
|
|
# is tmpfiles', and absent until that has run.
|
|
systemd.services.nginx = {
|
|
serviceConfig.ReadWritePaths = [ "-${egressDir}" ];
|
|
preStart = lib.mkBefore "rm -f ${egressSocket}";
|
|
};
|
|
|
|
# Reads the forwarder's client secret with the responder's store identity,
|
|
# the shape ./swarm-bao.nix's `swarm-bao-forwarder-oidc` has. The store
|
|
# and authelia may be a host away and this boot waits on neither, so a run
|
|
# that fails retries (./lib/store-retry.nix), and the collector restarts
|
|
# until the file lands.
|
|
systemd.services.${forwarderFetchUnit} = {
|
|
description = "fetch the metrics store forwarder's OIDC client secret from the swarm secret store";
|
|
after = [
|
|
"swarm-bao-pki.service"
|
|
"container@${cfg.machine}.service"
|
|
];
|
|
wants = [ "container@${cfg.machine}.service" ];
|
|
wantedBy = [ "multi-user.target" ];
|
|
path = [
|
|
deployCfg.bao.package
|
|
pkgs.coreutils
|
|
pkgs.systemd
|
|
];
|
|
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
|
serviceConfig = storeRetry.serviceConfig // {
|
|
# No `RemainAfterExit`: the timer below starts this again, and
|
|
# starting an active unit is a no-op.
|
|
Type = "oneshot";
|
|
SyslogIdentifier = forwarderFetchUnit;
|
|
TimeoutStartSec = 30;
|
|
};
|
|
environment = {
|
|
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
|
BAO_CLIENT_CERT = deployCfg.victoriametrics.authBaoClientCertFile;
|
|
BAO_CLIENT_KEY = deployCfg.victoriametrics.authBaoClientKeyFile;
|
|
}
|
|
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
|
|
BAO_CACERT = deployCfg.bao.serverCaFile;
|
|
};
|
|
script = ''
|
|
set -euo pipefail
|
|
|
|
${atomicWriteSecret}
|
|
${refreshConsumer}
|
|
|
|
err="$(mktemp)"
|
|
trap 'rm -f "$err"' EXIT
|
|
|
|
# `-token-only` answers on stdout and skips the token helper, which
|
|
# needs a `sh` this unit's `path` does not carry.
|
|
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
|
echo "could not log in to the swarm secret store with the metrics store responder's certificate." >&2
|
|
cat "$err" >&2
|
|
exit 1
|
|
fi
|
|
export BAO_TOKEN
|
|
|
|
if ! secret="$(bao kv get -field=value ${lib.escapeShellArg forwarderStoreSecretPath} 2>"$err")"; then
|
|
echo "the store did not return ${forwarderStoreSecretPath}: the forwarder has no client secret yet." >&2
|
|
cat "$err" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [ -z "$secret" ]; then
|
|
echo "the store returned an empty ${forwarderStoreSecretPath}." >&2
|
|
exit 1
|
|
fi
|
|
|
|
# `printf` inside `atomic_write_secret` is bash's own, so the value
|
|
# never becomes an argument in /proc.
|
|
install -d -m 0700 ${lib.escapeShellArg forwarderSecretDir}
|
|
if secret_differs ${lib.escapeShellArg forwarderSecret} "$secret"; then
|
|
atomic_write_secret 0400 root:root ${lib.escapeShellArg forwarderSecret} "$secret"
|
|
fi
|
|
|
|
# `LoadCredential` copies the file at start only: a rotated secret
|
|
# reaches the forwarder by restarting it.
|
|
refresh_consumer ${lib.escapeShellArg cfg.machine} opentelemetry-collector.service ${lib.escapeShellArg forwarderSecret}
|
|
'';
|
|
};
|
|
|
|
systemd.timers.${forwarderFetchUnit} =
|
|
storeRefreshTimer "re-read the metrics store forwarder's OIDC client secret from the swarm secret store";
|
|
|
|
containers.${cfg.machine} = {
|
|
autoStart = true;
|
|
ephemeral = false;
|
|
# No `--link-journal=host`: the journal stays in the guest, where the
|
|
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
|
|
inherit privateNetwork;
|
|
|
|
# A bind mount rather than a path both sides happen to know: a
|
|
# container's `/run` is an nspawn tmpfs the host cannot see into.
|
|
bindMounts.${socketDir} = {
|
|
hostPath = socketDir;
|
|
isReadOnly = false;
|
|
};
|
|
# Connecting to a socket is a write to it.
|
|
bindMounts.${egressDir} = {
|
|
hostPath = egressDir;
|
|
isReadOnly = false;
|
|
};
|
|
bindMounts.${forwarderSecretDir} = {
|
|
hostPath = forwarderSecretDir;
|
|
isReadOnly = true;
|
|
};
|
|
|
|
config =
|
|
{ config, pkgs, ... }:
|
|
{
|
|
imports = [
|
|
../container-modules/swarm-container.nix
|
|
../container-modules/swarm-journal-forwarder.nix
|
|
];
|
|
|
|
# Authenticated as the forwarder's own OIDC client, its secret
|
|
# fetched on the host. Both URLs are the relay's.
|
|
services.hyperhive.swarmJournalForwarder = {
|
|
enable = true;
|
|
serviceName = "victoriametrics";
|
|
endpoint = "${relayUrl}/${cfg.machine}";
|
|
clientId = forwarderClientId;
|
|
tokenUrl = "${relayUrl}/api/oidc/token";
|
|
telemetryPort = forwarderTelemetryPort;
|
|
hostSecretFile = forwarderSecret;
|
|
};
|
|
|
|
# The relay out of this netns, started by its first connection.
|
|
# systemd-socket-proxyd as root: the egress directory admits root
|
|
# and nginx only.
|
|
systemd.sockets.${relayUnit} = {
|
|
description = "loopback listener for the journal forwarder's way out of this container";
|
|
wantedBy = [ "sockets.target" ];
|
|
listenStreams = [ "127.0.0.1:${toString relayPort}" ];
|
|
};
|
|
systemd.services.${relayUnit} = {
|
|
description = "relay the journal forwarder's connections to the host's egress socket";
|
|
requires = [ "${relayUnit}.socket" ];
|
|
after = [ "${relayUnit}.socket" ];
|
|
serviceConfig = {
|
|
ExecStart = "${pkgs.systemd}/lib/systemd/systemd-socket-proxyd ${egressSocket}";
|
|
PrivateTmp = true;
|
|
NoNewPrivileges = true;
|
|
};
|
|
};
|
|
|
|
# No resolver unit: with no interface but loopback there is no
|
|
# nameserver to reach, and the store resolves no names.
|
|
services.hyperhive.swarmContainer = {
|
|
inherit privateNetwork;
|
|
writesOwnResolvConf = false;
|
|
};
|
|
|
|
# The socket's group is the store's primary group, so that group is
|
|
# nginx's gid. A static user, not nixpkgs' DynamicUser: a dynamic
|
|
# user's group is allocated per start, and nginx is never in it.
|
|
users.groups.gateway-nginx.gid = nginxGid;
|
|
users.users.victoriametrics = {
|
|
isSystemUser = true;
|
|
group = "gateway-nginx";
|
|
};
|
|
|
|
systemd.services.victoriametrics = {
|
|
serviceConfig = {
|
|
DynamicUser = lib.mkForce false;
|
|
User = "victoriametrics";
|
|
Group = "gateway-nginx";
|
|
# The store creates its socket under the process umask, so
|
|
# `0007` makes it `0770` and nginx's group can connect.
|
|
UMask = "0007";
|
|
};
|
|
# nixpkgs' readiness loop curls `http://<listenAddress>/ping`,
|
|
# which no `unix:` address answers.
|
|
postStart = lib.mkForce ''
|
|
until ${lib.getExe pkgs.curl} -s -o /dev/null --unix-socket ${socketPath} http://localhost/ping; do
|
|
sleep 1;
|
|
done
|
|
'';
|
|
};
|
|
|
|
# Moves the store's history out of the DynamicUser layout, where
|
|
# `/var/lib/<stateDir>` is a symlink into `/var/lib/private/`, to the
|
|
# plain directory the static user's StateDirectory is. Runs before
|
|
# the store on every start and does nothing once
|
|
# `/var/lib/private/<stateDir>` is gone. When both paths hold data
|
|
# it moves neither and the store starts on the plain one.
|
|
systemd.services.victoriametrics-state-migrate =
|
|
let
|
|
old = "/var/lib/private/${config.services.victoriametrics.stateDir}";
|
|
new = "/var/lib/${config.services.victoriametrics.stateDir}";
|
|
in
|
|
{
|
|
description = "move VictoriaMetrics' DynamicUser state directory to its static path";
|
|
before = [ "victoriametrics.service" ];
|
|
requiredBy = [ "victoriametrics.service" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
SyslogIdentifier = "victoriametrics-state-migrate";
|
|
};
|
|
path = [ pkgs.coreutils ];
|
|
script = ''
|
|
set -eu
|
|
old=${lib.escapeShellArg old}
|
|
new=${lib.escapeShellArg new}
|
|
if [ ! -d "$old" ] || [ -L "$old" ]; then
|
|
exit 0
|
|
fi
|
|
if [ -L "$new" ]; then
|
|
rm "$new"
|
|
elif [ -d "$new" ]; then
|
|
if [ -n "$(ls -A "$new")" ]; then
|
|
echo "victoriametrics-state-migrate: $old and $new both hold data; moving neither" >&2
|
|
exit 0
|
|
fi
|
|
rmdir "$new"
|
|
fi
|
|
mv "$old" "$new"
|
|
chown -R victoriametrics:gateway-nginx "$new"
|
|
echo "victoriametrics-state-migrate: moved $old to $new"
|
|
'';
|
|
};
|
|
|
|
services.victoriametrics = {
|
|
enable = true;
|
|
package = deployCfg.victoriametrics.package;
|
|
retentionPeriod = deployCfg.victoriametrics.retentionPeriod;
|
|
|
|
# ⚠️ A socket, not upstream's default `:8428` on every interface:
|
|
# the gateway is the only intended client, and this listener
|
|
# carries an unauthenticated write endpoint (see the OTLP note
|
|
# below).
|
|
listenAddress = "unix:${socketPath}";
|
|
};
|
|
};
|
|
};
|
|
};
|
|
|
|
# 🔑 OTLP ingest needs no flag. Measured against the pinned 1.146.0 rather
|
|
# than inferred from the module's option list, which has no OTLP switch and
|
|
# so reads as though the feature were absent: the running server answers
|
|
# `POST /opentelemetry/api/v1/push` with 200 (a nonexistent path answers
|
|
# 400, so that 200 means the route exists). The `-opentelemetry.*` flags
|
|
# only tune naming and limits, and are reachable via `extraOptions` if a
|
|
# deployment ever needs them.
|
|
#
|
|
# ⚠️ That endpoint is unauthenticated, which is why `listenAddress` above is
|
|
# a socket only the gateway can reach: the only route to it is the vhost's
|
|
# ingest location, which authenticates. The collector is still the only writer, but
|
|
# it now arrives by the swarm name rather than over loopback, because it need
|
|
# not share a host with this store.
|
|
}
|