901 lines
38 KiB
Nix
901 lines
38 KiB
Nix
# The swarm's metrics store: one VictoriaMetrics for the whole swarm, in a
|
||
# `swarm-victoriametrics` nixos-container.
|
||
#
|
||
# A LOCAL time-series database rather than only an external sink, and that is
|
||
# the point rather than a convenience: the swarm dashboard has to stay
|
||
# readable when the outside world is unreachable. Same failure-domain property
|
||
# the hive-status KV has — a view of the system must not depend on the system
|
||
# it is viewing being healthy.
|
||
#
|
||
# It is the collector that feeds this (the gateway OTEL collector), not the
|
||
# agents directly: one ingest point per swarm, authenticated there.
|
||
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
cfg = config.services.hyperhive.swarm.victoriametrics;
|
||
deployCfg = config.services.hyperhive.deploy;
|
||
hyperhiveCfg = config.services.hyperhive;
|
||
gatewayCfg = hyperhiveCfg.gateway;
|
||
|
||
# A netns of its own with no interface but loopback: the store makes no
|
||
# outbound calls, the gateway reaches it over `socketPath` below, and the
|
||
# journal forwarder leaves only through `egressSocket`, so nothing on the host
|
||
# or the hive bridge can dial its listener.
|
||
privateNetwork = true;
|
||
|
||
# Fixed, not an option: the directory's mode is what keeps the socket
|
||
# private, so it holds this one socket and nothing else.
|
||
socketDir = "/run/swarm-victoriametrics";
|
||
socketPath = "${socketDir}/victoriametrics.sock";
|
||
upstream = path: "http://unix:${socketPath}:${path}";
|
||
|
||
# The gid the gateway's nginx runs as. Only the number crosses into the
|
||
# container, which knows it as `gateway-nginx`.
|
||
nginxGid = config.ids.gids.nginx;
|
||
|
||
# The store's read API, which the vhost forwards to a caller holding
|
||
# Grafana's bearer, an agent's own bearer, or an operator's session. Each
|
||
# location's `proxy_pass` carries the path, so the store receives the path
|
||
# nginx matched, not the caller's unnormalised spelling.
|
||
#
|
||
# Grafana's Prometheus datasource reads the query, series, label, metadata,
|
||
# buildinfo, exemplar, rules and alerts routes; the store answers the last
|
||
# three with fixed placeholders. vmui (`uiLocations` below) is static files
|
||
# plus the query API and the remaining read-only routes here. Paths read off
|
||
# the pinned build's `app/vmselect/main.go` and vmui's `src/`.
|
||
readPaths = [
|
||
"/api/v1/query"
|
||
"/api/v1/query_range"
|
||
"/api/v1/query_exemplars"
|
||
"/api/v1/series"
|
||
"/api/v1/labels"
|
||
"/api/v1/metadata"
|
||
"/api/v1/rules"
|
||
"/api/v1/alerts"
|
||
"/api/v1/status/buildinfo"
|
||
"/api/v1/export"
|
||
"/api/v1/export/csv"
|
||
"/api/v1/status/tsdb"
|
||
"/api/v1/status/metric_names_stats"
|
||
"/api/v1/status/top_queries"
|
||
"/api/v1/status/active_queries"
|
||
"/expand-with-exprs"
|
||
"/prettify-query"
|
||
"/metric-relabel-debug"
|
||
];
|
||
# The store serves only `<name>/values` under `/api/v1/label/`.
|
||
readPrefixes = [ "/api/v1/label/" ];
|
||
|
||
# Bare `auth_request`, no login redirect: a machine caller handed
|
||
# `error_page 401 =302` follows it and reads authelia's login page as a 200
|
||
# holding no series. `/__metrics_read_auth` picks the check from the
|
||
# caller's `Authorization` (`appendHttpConfig` below).
|
||
readAuth = ''
|
||
auth_request /__metrics_read_auth;
|
||
'';
|
||
readLocations =
|
||
let
|
||
to = path: {
|
||
proxyPass = upstream path;
|
||
extraConfig = readAuth;
|
||
};
|
||
in
|
||
lib.listToAttrs (
|
||
map (p: lib.nameValuePair "= ${p}" (to p)) readPaths
|
||
++ map (p: lib.nameValuePair "^~ ${p}" (to p)) readPrefixes
|
||
);
|
||
|
||
# The same three lines as `swarmAuthRequest` in ./swarm-victorialogs.nix,
|
||
# for the same browser: an operator without a session is sent to log in.
|
||
swarmAuthRequest = ''
|
||
auth_request /__metrics_authelia;
|
||
auth_request_set $target_url $scheme://$http_host$request_uri;
|
||
error_page 401 =302 https://${hyperhiveCfg.swarm.authelia.domain}/?rd=$target_url;
|
||
'';
|
||
# vmui's static files and UI config. Only a browser loads these, and its
|
||
# queries then go to the read locations above with the same session cookie.
|
||
uiLocations = {
|
||
"= /vmui" = {
|
||
proxyPass = upstream "/vmui";
|
||
extraConfig = swarmAuthRequest;
|
||
};
|
||
"^~ /vmui/" = {
|
||
proxyPass = upstream "/vmui/";
|
||
extraConfig = swarmAuthRequest;
|
||
};
|
||
};
|
||
|
||
# The `auth_request` responder for Grafana's bearer (`swarm-metrics-auth`).
|
||
# Its socket's directory holds that socket alone and is traverse-only for
|
||
# others, the shape `swarm-logs-auth` in ./swarm-victorialogs.nix has.
|
||
metricsAuthUnit = "swarm-metrics-auth";
|
||
metricsAuthSocket = "/run/${metricsAuthUnit}/auth.sock";
|
||
metricsAuthCredential = id: "/run/credentials/${metricsAuthUnit}.service/${id}";
|
||
# Credential id → host path of the responder's store identity.
|
||
metricsAuthStoreFiles = {
|
||
"bao-client.pem" = deployCfg.victoriametrics.authBaoClientCertFile;
|
||
"bao-client-key.pem" = deployCfg.victoriametrics.authBaoClientKeyFile;
|
||
}
|
||
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
|
||
"bao-ca.pem" = deployCfg.bao.serverCaFile;
|
||
};
|
||
baoCfg = hyperhiveCfg.swarm.bao;
|
||
# The path Grafana's host creates the bearer at, under the KV mount. The
|
||
# same leaf as `grafanaMetricsBearerLeaf` in ./swarm-bao.nix.
|
||
grafanaBearerStorePath = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer";
|
||
|
||
# The agents' check on the read API: a `swarm-logs-auth` binary, logging in
|
||
# under this host's `swarm-metrics-auth` leaf, whose grant covers every
|
||
# agent's queue secret. The log store's host runs `swarm-metrics-auth` under
|
||
# its `swarm-logs-auth` leaf the same way.
|
||
agentAuthUnit = "swarm-metrics-agent-auth";
|
||
agentAuthSocket = "/run/${agentAuthUnit}/auth.sock";
|
||
agentAuthCredential = id: "/run/credentials/${agentAuthUnit}.service/${id}";
|
||
|
||
# ── the container's journal forwarder ───────────────────────────────
|
||
#
|
||
# With no interface but loopback, the forwarder reaches nothing by name. A
|
||
# relay in the container listens on `relayPort` and forwards each connection
|
||
# to `egressSocket`, a socket this host's nginx listens on and binds into
|
||
# the container. nginx sends `/<machine>/` to the swarm collector's route
|
||
# (`serviceProducers.swarm-victoriametrics` in ./swarm-otel-service.nix) and
|
||
# the token endpoint to authelia, both by name and verified. The collector
|
||
# library dials no unix socket itself, so the relay is needed.
|
||
#
|
||
# The client secret is fetched on the host, under the responder's store
|
||
# identity, and bound in read-only: the container cannot reach the store.
|
||
forwarderClientId = cfg.otel.clientId;
|
||
gatewayPortSuffix = lib.optionalString (
|
||
gatewayCfg.httpsPort != 443
|
||
) ":${toString gatewayCfg.httpsPort}";
|
||
otelDomain = hyperhiveCfg.swarm.otel.domain;
|
||
autheliaDomain = hyperhiveCfg.swarm.authelia.domain;
|
||
|
||
# Its own directory, not `socketDir`: that one's mode admits nginx to the
|
||
# store's listener, and this one admits the container to nginx's.
|
||
egressDir = "/run/swarm-victoriametrics-egress";
|
||
egressSocket = "${egressDir}/egress.sock";
|
||
relayUnit = "swarm-victoriametrics-egress-relay";
|
||
# Loopback in the container's own netns.
|
||
relayPort = 4380;
|
||
relayUrl = "http://127.0.0.1:${toString relayPort}";
|
||
forwarderTelemetryPort = 8900;
|
||
|
||
# The forwarder's way out, see `egressSocket`. Every other path is refused,
|
||
# so the container reaches the two routes its forwarder uses and nothing
|
||
# else on the network.
|
||
egressServer = ''
|
||
server {
|
||
listen unix:${egressSocket};
|
||
|
||
location / {
|
||
return 403;
|
||
}
|
||
|
||
location ^~ /${cfg.machine}/ {
|
||
proxy_pass https://${otelDomain}${gatewayPortSuffix};
|
||
${gatewayCfg.lib.verifiedProxyTo otelDomain}
|
||
}
|
||
|
||
location = /api/oidc/token {
|
||
proxy_pass https://${autheliaDomain};
|
||
${gatewayCfg.lib.verifiedProxyTo autheliaDomain}
|
||
}
|
||
}
|
||
'';
|
||
|
||
forwarderFetchUnit = "swarm-victoriametrics-forwarder-oidc";
|
||
forwarderStoreSecretPath = "secret/swarm/services/${forwarderClientId}/oidc/client";
|
||
# On the host's /run (tmpfs), and seen read-only in the container at the
|
||
# same path. The file is not created ahead of the fetch: `LoadCredential`
|
||
# refuses a missing source, so the forwarder stays down until the store
|
||
# answers and then resumes from its saved read cursor.
|
||
forwarderSecretDir = "/run/${forwarderFetchUnit}";
|
||
forwarderSecret = "${forwarderSecretDir}/${forwarderClientId}.secret";
|
||
|
||
storeRetry = import ./lib/store-retry.nix { };
|
||
storeRefreshTimer = import ./lib/store-refresh-timer.nix { };
|
||
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
|
||
refreshConsumer = import ./lib/refresh-consumer.nix { };
|
||
in
|
||
{
|
||
# What the store IS from any hive's point of view, the name it answers on,
|
||
# is `swarm.victoriametrics` in ./swarm-victoriametrics-service.nix.
|
||
# `enable`, `package` and `retentionPeriod` are decisions of the host that
|
||
# runs it and live under `deploy.*`.
|
||
|
||
# Retention is a property of the store this host runs, not something the
|
||
# swarm has to agree on: it is read only where the container is defined,
|
||
# and a hive that is a *client* of the metrics store never consults it.
|
||
options.services.hyperhive.deploy.victoriametrics.package = lib.mkOption {
|
||
type = lib.types.package;
|
||
default = pkgs.victoriametrics;
|
||
defaultText = lib.literalExpression "pkgs.victoriametrics";
|
||
description = "VictoriaMetrics package to run.";
|
||
};
|
||
|
||
options.services.hyperhive.deploy.victoriametrics.retentionPeriod = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "5y";
|
||
example = "90d";
|
||
description = ''
|
||
How long samples are kept.
|
||
|
||
Deliberately a high default rather than a required option: the two
|
||
failure directions are not symmetric. Too long fills a disk, which
|
||
is visible and recoverable by lowering this; too short **destroys
|
||
history**, silently and permanently. So the safe default is generous
|
||
and an operator lowers it once they have measured how fast this swarm
|
||
actually accumulates data.
|
||
'';
|
||
};
|
||
|
||
options.services.hyperhive.deploy.victoriametrics.authPackage = lib.mkOption {
|
||
type = lib.types.package;
|
||
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-metrics-auth";
|
||
description = ''
|
||
The `swarm-metrics-auth` package: the gateway's `auth_request` responder
|
||
that checks the bearer Grafana's metrics datasource presents on the
|
||
store's query API against the secret store.
|
||
'';
|
||
};
|
||
|
||
options.services.hyperhive.deploy.victoriametrics.agentAuthPackage = lib.mkOption {
|
||
type = lib.types.package;
|
||
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-logs-auth";
|
||
description = ''
|
||
The `swarm-logs-auth` package, run as `swarm-metrics-agent-auth`: the
|
||
gateway's `auth_request` responder that checks an agent's
|
||
`swarm-agent.<agent>.<secret>` bearer on the read API against the
|
||
secret store.
|
||
|
||
⚠️ Not folded into `authPackage` above: that one is
|
||
`swarm-metrics-auth`, a different binary. Same split as
|
||
{option}`services.hyperhive.deploy.nats.authPackage`.
|
||
'';
|
||
};
|
||
|
||
options.services.hyperhive.deploy.victoriametrics.authBaoClientCertFile = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/var/lib/swarm-bao-pki/metrics-auth.pem";
|
||
description = ''
|
||
Client certificate `swarm-metrics-auth` presents to the swarm's secret
|
||
store to read Grafana's metrics bearer, and `swarm-metrics-agent-auth`
|
||
to read agents' queue secrets. Its subject must be
|
||
{option}`services.hyperhive.deploy.bao.metricsAuthCommonName`.
|
||
|
||
Required when `deploy.victoriametrics.enable` is set; unset fails at
|
||
eval. ./glue-metrics-auth-bao-identity.nix points it at the leaf
|
||
./glue-bao-tls.nix mints on a host that runs the store; on any other
|
||
metrics-store host it is a file the operator places.
|
||
|
||
A path, never a value.
|
||
'';
|
||
};
|
||
|
||
options.services.hyperhive.deploy.victoriametrics.authBaoClientKeyFile = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/var/lib/swarm-bao-pki/metrics-auth-key.pem";
|
||
description = ''
|
||
Private key for {option}`services.hyperhive.deploy.victoriametrics.authBaoClientCertFile`.
|
||
A path, never a value.
|
||
'';
|
||
};
|
||
|
||
options.services.hyperhive.deploy.victoriametrics.agentAuthVerdictTtl = lib.mkOption {
|
||
type = lib.types.ints.unsigned;
|
||
default = 60;
|
||
description = ''
|
||
Seconds `swarm-metrics-agent-auth` grants an agent bearer it has
|
||
verified again without reading the secret store. A denial is never
|
||
remembered.
|
||
|
||
This is also how long a re-minted or deleted queue secret keeps working
|
||
at the metrics store's read API after its last successful check. `0`
|
||
checks the store on every query.
|
||
'';
|
||
};
|
||
|
||
options.services.hyperhive.deploy.victoriametrics.agentAuthRate = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "50r/s";
|
||
example = "30r/m";
|
||
description = ''
|
||
nginx `limit_req` rate, per client address, of read API requests
|
||
carrying a `swarm-agent.` bearer, which nginx checks with
|
||
`swarm-metrics-agent-auth`. Each such request that the responder has
|
||
not granted within
|
||
{option}`services.hyperhive.deploy.victoriametrics.agentAuthVerdictTtl`
|
||
costs a secret store login and read, whether the bearer is right or not.
|
||
|
||
Keyed by client address, so every agent on one host shares one budget.
|
||
Requests over the rate and its burst are answered 403.
|
||
'';
|
||
};
|
||
|
||
options.services.hyperhive.deploy.victoriametrics.agentAuthBurst = lib.mkOption {
|
||
type = lib.types.ints.positive;
|
||
default = 500;
|
||
description = ''
|
||
Requests a client address may make above
|
||
{option}`services.hyperhive.deploy.victoriametrics.agentAuthRate` before
|
||
nginx answers 403, served without delay.
|
||
'';
|
||
};
|
||
|
||
config = lib.mkIf deployCfg.victoriametrics.enable {
|
||
assertions = [
|
||
{
|
||
assertion = lib.versionAtLeast (lib.getVersion deployCfg.victoriametrics.package) "1.153.0";
|
||
message = ''
|
||
services.hyperhive.deploy.victoriametrics.package must be
|
||
VictoriaMetrics 1.153.0 or later (this one is
|
||
${lib.getVersion deployCfg.victoriametrics.package}).
|
||
|
||
The gateway reaches the store over a unix socket, and only from
|
||
1.153.0 does VictoriaMetrics create that socket under the process
|
||
umask. An older build creates it 0600 whatever the umask, so nginx
|
||
cannot connect and every read, push and scrape fails with 502.
|
||
'';
|
||
}
|
||
{
|
||
assertion =
|
||
deployCfg.victoriametrics.authBaoClientCertFile != null
|
||
&& deployCfg.victoriametrics.authBaoClientKeyFile != null;
|
||
message = ''
|
||
services.hyperhive.deploy.victoriametrics.authBaoClientCertFile and
|
||
authBaoClientKeyFile must both be set.
|
||
|
||
swarm-metrics-auth reads Grafana's metrics bearer, and
|
||
swarm-metrics-agent-auth agents' queue secrets, from the swarm's
|
||
secret store with this certificate. They default to the leaf the
|
||
store mints where services.hyperhive.deploy.bao.enable is set on
|
||
this host; elsewhere, issue a leaf whose CN is
|
||
services.hyperhive.deploy.bao.metricsAuthCommonName and name it in
|
||
both options.
|
||
'';
|
||
}
|
||
{
|
||
assertion =
|
||
(hyperhiveCfg.swarm.otel.serviceProducers.${cfg.machine}.clientId or null) == forwarderClientId;
|
||
message = ''
|
||
services.hyperhive.swarm.otel.serviceProducers has no entry
|
||
"${cfg.machine}" with clientId "${forwarderClientId}", so the swarm
|
||
collector has no receiver for the metrics store's journal forwarder.
|
||
Declare it in swarm-otel-service.nix.
|
||
'';
|
||
}
|
||
];
|
||
|
||
# The gateway name, inside `deployCfg.victoriametrics.enable` — that
|
||
# guard is the load-bearing part. Every hive in a swarm may know this
|
||
# store exists, but only the host that RUNS it may claim the name; a
|
||
# client hive declaring the vhost would answer for a service it does not
|
||
# have.
|
||
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
||
|
||
# This host serves the vhost, and the hive's dnsmasq answers its name.
|
||
services.hyperhive.gateway.enable = lib.mkDefault true;
|
||
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
||
|
||
# Which check a read location's `auth_request` gets. `auth_request` takes a
|
||
# fixed URI, so it names `/__metrics_read_auth`, which rewrites to the
|
||
# target this map picks: an agent's `swarm-agent.` bearer goes to the
|
||
# agents' check, any other bearer to Grafana's, anything else (an
|
||
# operator's session cookie) to authelia. A map tries its regexes in
|
||
# order, so the agent arm comes first. After the forwarder's egress
|
||
# listener, `egressServer`.
|
||
services.nginx.appendHttpConfig = egressServer + ''
|
||
map $http_authorization $swarm_metrics_read_auth_target {
|
||
"~^Bearer swarm-agent\." /__metrics_agent_auth;
|
||
"~^Bearer " /__metrics_grafana_auth;
|
||
default /__metrics_authelia;
|
||
}
|
||
limit_req_zone $binary_remote_addr zone=swarm_metrics_agent_auth:1m rate=${deployCfg.victoriametrics.agentAuthRate};
|
||
'';
|
||
|
||
# `forceSSL`: authelia answers a plain-http auth subrequest
|
||
# with 400, which `auth_request` cannot read as anything but a broken
|
||
# check. Same as ./swarm-victorialogs.nix's vhost.
|
||
services.hyperhive.gateway.vhosts."${cfg.domain}" = {
|
||
forceSSL = true;
|
||
locations =
|
||
readLocations
|
||
// uiLocations
|
||
// {
|
||
# Nothing reaches the store unless a location here names it. The
|
||
# store serves writes and admin calls (`/api/v1/import`,
|
||
# `/api/v1/admin/tsdb/delete_series`, ...) on the same listener as
|
||
# its queries, and authenticates none of them.
|
||
"/".return = "403";
|
||
|
||
# The swarm UI's quick link opens the bare domain. The store's own
|
||
# index page links mostly to paths refused above, so send it to vmui.
|
||
"= /".return = "302 /vmui/";
|
||
|
||
# The swarm collector's ingest route. `=` so it outranks the `/`
|
||
# prefix above.
|
||
#
|
||
# ⚠️ No `error_page 401 =302` here, and its absence is the point: a
|
||
# redirect is right for a browser and wrong for a pusher, which would
|
||
# follow it and POST its batch at a login page that answers 200 —
|
||
# ingest reporting healthy while storing nothing.
|
||
"= /opentelemetry/api/v1/push" = {
|
||
proxyPass = upstream "/opentelemetry/api/v1/push";
|
||
extraConfig = ''
|
||
auth_request /__metrics_authelia;
|
||
'';
|
||
};
|
||
|
||
# The store's own health, for the swarm collector wherever it runs:
|
||
# served on the query listener, so no exporter and no extra port. `=`
|
||
# for the same reason as the push route, and no `error_page` for the
|
||
# same reason: a scraper would parse a login page as metrics.
|
||
#
|
||
# The collector requests exactly this URL, declared as
|
||
# `publishedScrapeTargets.victoriametrics` in ./swarm-otel-service.nix;
|
||
# the two must agree, since that string is also the token's audience.
|
||
"= /metrics" = {
|
||
proxyPass = upstream "/metrics";
|
||
extraConfig = ''
|
||
auth_request /__metrics_authelia;
|
||
'';
|
||
};
|
||
|
||
# `rewrite … last` re-matches locations, so the subrequest lands in
|
||
# one of the two internal locations named by the map.
|
||
"= /__metrics_read_auth" = {
|
||
extraConfig = ''
|
||
internal;
|
||
rewrite ^ $swarm_metrics_read_auth_target last;
|
||
'';
|
||
};
|
||
|
||
# Grafana's bearer, checked by `swarm-metrics-auth`. The
|
||
# `Authorization` header rides the subrequest unchanged. No rate limit
|
||
# in front: the responder reads the store at most once per its cache
|
||
# TTL, whatever the bearers.
|
||
"= /__metrics_grafana_auth" = {
|
||
proxyPass = "http://unix:${metricsAuthSocket}:/";
|
||
extraConfig = ''
|
||
internal;
|
||
proxy_pass_request_body off;
|
||
proxy_set_header Content-Length "";
|
||
'';
|
||
};
|
||
|
||
# An agent's bearer, checked by `swarm-metrics-agent-auth`. Any
|
||
# well-formed agent bearer the responder has not recently granted
|
||
# costs a store login and read, so the rate per client address is
|
||
# capped here. 403 when over: `auth_request` passes only 401 and 403
|
||
# through, and `swarm-metrics` retries a 401.
|
||
"= /__metrics_agent_auth" = {
|
||
proxyPass = "http://unix:${agentAuthSocket}:/";
|
||
extraConfig = ''
|
||
internal;
|
||
limit_req zone=swarm_metrics_agent_auth burst=${toString deployCfg.victoriametrics.agentAuthBurst} nodelay;
|
||
limit_req_status 403;
|
||
proxy_pass_request_body off;
|
||
proxy_set_header Content-Length "";
|
||
'';
|
||
};
|
||
|
||
# The authelia subrequest. For the collector's two locations above it
|
||
# admits only the collector's client; for the read and vmui locations
|
||
# it admits an operator's session (`metricsStoreRules` in
|
||
# ./swarm-authelia.nix). Same target and header set as the sibling log
|
||
# store's, which took them from `swarm-ui.nix` — `X-Original-URL` and
|
||
# `X-Original-Method` are what authelia's auth-request implementation
|
||
# reads, and the address it compares the token's audience against.
|
||
"= /__metrics_authelia" = {
|
||
proxyPass = "https://${hyperhiveCfg.swarm.authelia.domain}/api/authz/auth-request";
|
||
# nixpkgs appends its OWN `Host $host` after extraConfig, which
|
||
# would override verifiedProxyTo's — see the comment on
|
||
# verifiedProxyTo in hive-gateway/vhost-lib.nix.
|
||
recommendedProxySettings = false;
|
||
extraConfig = ''
|
||
internal;
|
||
${gatewayCfg.lib.verifiedProxyTo hyperhiveCfg.swarm.authelia.domain}
|
||
proxy_pass_request_body off;
|
||
proxy_set_header Content-Length "";
|
||
proxy_set_header X-Original-Method $request_method;
|
||
proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
|
||
proxy_set_header X-Forwarded-Proto $scheme;
|
||
proxy_set_header X-Forwarded-Host $http_host;
|
||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||
'';
|
||
};
|
||
};
|
||
};
|
||
|
||
# The socket directory, created host-side before the container starts:
|
||
# nixos-container refuses to start when a bind source is missing.
|
||
#
|
||
# Root-owned and group nginx, `0770`: the store creates its socket here
|
||
# through the group (its uid is allocated inside the container, so the
|
||
# host cannot name it), and the gateway traverses it the same way.
|
||
# Everything else is kept out.
|
||
systemd.tmpfiles.rules = [
|
||
"d ${socketDir} 0770 root ${toString nginxGid} - -"
|
||
# The relay in the container connects as root; nothing else on the host
|
||
# but nginx may.
|
||
"d ${egressDir} 0750 ${config.services.nginx.user} ${config.services.nginx.group} - -"
|
||
# The forwarder secret's bind source, so it exists when nspawn sets the
|
||
# mount up. Not the file: see `forwarderSecretDir`.
|
||
"d ${forwarderSecretDir} 0700 root root -"
|
||
];
|
||
|
||
# Grafana's check on the read API. On the host rather than in the store's
|
||
# container: it needs nothing of the store, only nginx's reach to its
|
||
# socket and its own store identity.
|
||
users.users.${metricsAuthUnit} = {
|
||
isSystemUser = true;
|
||
group = metricsAuthUnit;
|
||
description = "swarm metrics store auth_request responder";
|
||
};
|
||
users.groups.${metricsAuthUnit} = { };
|
||
|
||
systemd.services.${metricsAuthUnit} = {
|
||
description = "swarm metrics store auth_request responder for Grafana's bearer";
|
||
wantedBy = [ "multi-user.target" ];
|
||
# Where the store's PKI is minted on this host, the leaf is one of its
|
||
# files. Ordering only: elsewhere the unit does not exist.
|
||
after = [
|
||
"network.target"
|
||
"swarm-bao-pki.service"
|
||
];
|
||
serviceConfig = {
|
||
ExecStart = lib.concatStringsSep " " [
|
||
"${deployCfg.victoriametrics.authPackage}/bin/swarm-metrics-auth"
|
||
"--socket ${metricsAuthSocket}"
|
||
# The role ./swarm-bao.nix writes on the store's `cert` mount.
|
||
"--store-cert-role swarm-metrics-auth"
|
||
"--store-path ${grafanaBearerStorePath}"
|
||
];
|
||
# Paths only: `argv` is world-readable via /proc/<pid>/cmdline.
|
||
LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") metricsAuthStoreFiles;
|
||
# ⚠️ A static user, never `DynamicUser`: with `RuntimeDirectoryPreserve`
|
||
# systemd puts a dynamic user's runtime directory under
|
||
# `/run/private/` (0700 root:root), which nginx cannot traverse.
|
||
User = metricsAuthUnit;
|
||
Group = metricsAuthUnit;
|
||
RuntimeDirectory = metricsAuthUnit;
|
||
# Traverse-only for others, so nginx reaches the socket path.
|
||
RuntimeDirectoryMode = "0751";
|
||
# The path never vanishes from under a running nginx; the responder
|
||
# unlinks a stale socket as it starts.
|
||
RuntimeDirectoryPreserve = "yes";
|
||
Restart = "on-failure";
|
||
RestartSec = "5s";
|
||
SyslogIdentifier = metricsAuthUnit;
|
||
ProtectSystem = "strict";
|
||
ProtectHome = true;
|
||
PrivateTmp = "disconnected";
|
||
PrivateDevices = true;
|
||
NoNewPrivileges = true;
|
||
RestrictSUIDSGID = true;
|
||
RemoveIPC = true;
|
||
};
|
||
environment = {
|
||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||
BAO_CLIENT_CERT = metricsAuthCredential "bao-client.pem";
|
||
BAO_CLIENT_KEY = metricsAuthCredential "bao-client-key.pem";
|
||
}
|
||
// lib.optionalAttrs (metricsAuthStoreFiles ? "bao-ca.pem") {
|
||
BAO_CACERT = metricsAuthCredential "bao-ca.pem";
|
||
};
|
||
};
|
||
|
||
# The agents' check on the read API, see `agentAuthUnit`.
|
||
users.users.${agentAuthUnit} = {
|
||
isSystemUser = true;
|
||
group = agentAuthUnit;
|
||
description = "swarm metrics store auth_request responder for agent credentials";
|
||
};
|
||
users.groups.${agentAuthUnit} = { };
|
||
|
||
systemd.services.${agentAuthUnit} = {
|
||
description = "swarm metrics store auth_request responder for agent credentials";
|
||
wantedBy = [ "multi-user.target" ];
|
||
after = [
|
||
"network.target"
|
||
"swarm-bao-pki.service"
|
||
];
|
||
serviceConfig = {
|
||
ExecStart = lib.concatStringsSep " " [
|
||
"${deployCfg.victoriametrics.agentAuthPackage}/bin/swarm-logs-auth"
|
||
"--socket ${agentAuthSocket}"
|
||
"--store-cert-role swarm-metrics-auth"
|
||
"--verdict-ttl ${toString deployCfg.victoriametrics.agentAuthVerdictTtl}"
|
||
];
|
||
LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") metricsAuthStoreFiles;
|
||
# ⚠️ A static user, never `DynamicUser`: see the `swarm-metrics-auth` unit above.
|
||
User = agentAuthUnit;
|
||
Group = agentAuthUnit;
|
||
RuntimeDirectory = agentAuthUnit;
|
||
RuntimeDirectoryMode = "0751";
|
||
RuntimeDirectoryPreserve = "yes";
|
||
Restart = "on-failure";
|
||
RestartSec = "5s";
|
||
SyslogIdentifier = agentAuthUnit;
|
||
ProtectSystem = "strict";
|
||
ProtectHome = true;
|
||
PrivateTmp = "disconnected";
|
||
PrivateDevices = true;
|
||
NoNewPrivileges = true;
|
||
RestrictSUIDSGID = true;
|
||
RemoveIPC = true;
|
||
};
|
||
environment = {
|
||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||
BAO_CLIENT_CERT = agentAuthCredential "bao-client.pem";
|
||
BAO_CLIENT_KEY = agentAuthCredential "bao-client-key.pem";
|
||
}
|
||
// lib.optionalAttrs (metricsAuthStoreFiles ? "bao-ca.pem") {
|
||
BAO_CACERT = agentAuthCredential "bao-ca.pem";
|
||
};
|
||
};
|
||
|
||
# nginx runs as its own user under `ProtectSystem=strict`, so it can only
|
||
# create the socket where it may write. It removes the socket on a clean
|
||
# exit only; a stale one after a crash fails every later `bind`, and with
|
||
# it the whole gateway, so it goes before each start. `-`: the directory
|
||
# is tmpfiles', and absent until that has run.
|
||
systemd.services.nginx = {
|
||
serviceConfig.ReadWritePaths = [ "-${egressDir}" ];
|
||
preStart = lib.mkBefore "rm -f ${egressSocket}";
|
||
};
|
||
|
||
# Reads the forwarder's client secret with the responder's store identity,
|
||
# the shape ./swarm-bao.nix's `swarm-bao-forwarder-oidc` has. The store
|
||
# and authelia may be a host away and this boot waits on neither, so a run
|
||
# that fails retries (./lib/store-retry.nix), and the collector restarts
|
||
# until the file lands.
|
||
systemd.services.${forwarderFetchUnit} = {
|
||
description = "fetch the metrics store forwarder's OIDC client secret from the swarm secret store";
|
||
after = [
|
||
"swarm-bao-pki.service"
|
||
"container@${cfg.machine}.service"
|
||
];
|
||
wants = [ "container@${cfg.machine}.service" ];
|
||
wantedBy = [ "multi-user.target" ];
|
||
path = [
|
||
deployCfg.bao.package
|
||
pkgs.coreutils
|
||
pkgs.systemd
|
||
];
|
||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||
serviceConfig = storeRetry.serviceConfig // {
|
||
# No `RemainAfterExit`: the timer below starts this again, and
|
||
# starting an active unit is a no-op.
|
||
Type = "oneshot";
|
||
SyslogIdentifier = forwarderFetchUnit;
|
||
TimeoutStartSec = 30;
|
||
};
|
||
environment = {
|
||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||
BAO_CLIENT_CERT = deployCfg.victoriametrics.authBaoClientCertFile;
|
||
BAO_CLIENT_KEY = deployCfg.victoriametrics.authBaoClientKeyFile;
|
||
}
|
||
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
|
||
BAO_CACERT = deployCfg.bao.serverCaFile;
|
||
};
|
||
script = ''
|
||
set -euo pipefail
|
||
|
||
${atomicWriteSecret}
|
||
${refreshConsumer}
|
||
|
||
err="$(mktemp)"
|
||
trap 'rm -f "$err"' EXIT
|
||
|
||
# `-token-only` answers on stdout and skips the token helper, which
|
||
# needs a `sh` this unit's `path` does not carry.
|
||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||
echo "could not log in to the swarm secret store with the metrics store responder's certificate." >&2
|
||
cat "$err" >&2
|
||
exit 1
|
||
fi
|
||
export BAO_TOKEN
|
||
|
||
if ! secret="$(bao kv get -field=value ${lib.escapeShellArg forwarderStoreSecretPath} 2>"$err")"; then
|
||
echo "the store did not return ${forwarderStoreSecretPath}: the forwarder has no client secret yet." >&2
|
||
cat "$err" >&2
|
||
exit 1
|
||
fi
|
||
|
||
if [ -z "$secret" ]; then
|
||
echo "the store returned an empty ${forwarderStoreSecretPath}." >&2
|
||
exit 1
|
||
fi
|
||
|
||
# `printf` inside `atomic_write_secret` is bash's own, so the value
|
||
# never becomes an argument in /proc.
|
||
install -d -m 0700 ${lib.escapeShellArg forwarderSecretDir}
|
||
if secret_differs ${lib.escapeShellArg forwarderSecret} "$secret"; then
|
||
atomic_write_secret 0400 root:root ${lib.escapeShellArg forwarderSecret} "$secret"
|
||
fi
|
||
|
||
# `LoadCredential` copies the file at start only: a rotated secret
|
||
# reaches the forwarder by restarting it.
|
||
refresh_consumer ${lib.escapeShellArg cfg.machine} opentelemetry-collector.service ${lib.escapeShellArg forwarderSecret}
|
||
'';
|
||
};
|
||
|
||
systemd.timers.${forwarderFetchUnit} =
|
||
storeRefreshTimer "re-read the metrics store forwarder's OIDC client secret from the swarm secret store";
|
||
|
||
containers.${cfg.machine} = {
|
||
autoStart = true;
|
||
ephemeral = false;
|
||
# No `--link-journal=host`: the journal stays in the guest, where the
|
||
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
|
||
inherit privateNetwork;
|
||
|
||
# A bind mount rather than a path both sides happen to know: a
|
||
# container's `/run` is an nspawn tmpfs the host cannot see into.
|
||
bindMounts.${socketDir} = {
|
||
hostPath = socketDir;
|
||
isReadOnly = false;
|
||
};
|
||
# Connecting to a socket is a write to it.
|
||
bindMounts.${egressDir} = {
|
||
hostPath = egressDir;
|
||
isReadOnly = false;
|
||
};
|
||
bindMounts.${forwarderSecretDir} = {
|
||
hostPath = forwarderSecretDir;
|
||
isReadOnly = true;
|
||
};
|
||
|
||
config =
|
||
{ config, pkgs, ... }:
|
||
{
|
||
imports = [
|
||
../container-modules/swarm-container.nix
|
||
../container-modules/swarm-journal-forwarder.nix
|
||
];
|
||
|
||
# Authenticated as the forwarder's own OIDC client, its secret
|
||
# fetched on the host. Both URLs are the relay's.
|
||
services.hyperhive.swarmJournalForwarder = {
|
||
enable = true;
|
||
serviceName = "victoriametrics";
|
||
endpoint = "${relayUrl}/${cfg.machine}";
|
||
clientId = forwarderClientId;
|
||
tokenUrl = "${relayUrl}/api/oidc/token";
|
||
telemetryPort = forwarderTelemetryPort;
|
||
hostSecretFile = forwarderSecret;
|
||
};
|
||
|
||
# The relay out of this netns, started by its first connection.
|
||
# systemd-socket-proxyd as root: the egress directory admits root
|
||
# and nginx only.
|
||
systemd.sockets.${relayUnit} = {
|
||
description = "loopback listener for the journal forwarder's way out of this container";
|
||
wantedBy = [ "sockets.target" ];
|
||
listenStreams = [ "127.0.0.1:${toString relayPort}" ];
|
||
};
|
||
systemd.services.${relayUnit} = {
|
||
description = "relay the journal forwarder's connections to the host's egress socket";
|
||
requires = [ "${relayUnit}.socket" ];
|
||
after = [ "${relayUnit}.socket" ];
|
||
serviceConfig = {
|
||
ExecStart = "${pkgs.systemd}/lib/systemd/systemd-socket-proxyd ${egressSocket}";
|
||
PrivateTmp = true;
|
||
NoNewPrivileges = true;
|
||
};
|
||
};
|
||
|
||
# No resolver unit: with no interface but loopback there is no
|
||
# nameserver to reach, and the store resolves no names.
|
||
services.hyperhive.swarmContainer = {
|
||
inherit privateNetwork;
|
||
writesOwnResolvConf = false;
|
||
};
|
||
|
||
# The socket's group is the store's primary group, so that group is
|
||
# nginx's gid. A static user, not nixpkgs' DynamicUser: a dynamic
|
||
# user's group is allocated per start, and nginx is never in it.
|
||
users.groups.gateway-nginx.gid = nginxGid;
|
||
users.users.victoriametrics = {
|
||
isSystemUser = true;
|
||
group = "gateway-nginx";
|
||
};
|
||
|
||
systemd.services.victoriametrics = {
|
||
serviceConfig = {
|
||
DynamicUser = lib.mkForce false;
|
||
User = "victoriametrics";
|
||
Group = "gateway-nginx";
|
||
# The store creates its socket under the process umask, so
|
||
# `0007` makes it `0770` and nginx's group can connect.
|
||
UMask = "0007";
|
||
};
|
||
# nixpkgs' readiness loop curls `http://<listenAddress>/ping`,
|
||
# which no `unix:` address answers.
|
||
postStart = lib.mkForce ''
|
||
until ${lib.getExe pkgs.curl} -s -o /dev/null --unix-socket ${socketPath} http://localhost/ping; do
|
||
sleep 1;
|
||
done
|
||
'';
|
||
};
|
||
|
||
# Moves the store's history out of the DynamicUser layout, where
|
||
# `/var/lib/<stateDir>` is a symlink into `/var/lib/private/`, to the
|
||
# plain directory the static user's StateDirectory is. Runs before
|
||
# the store on every start and does nothing once
|
||
# `/var/lib/private/<stateDir>` is gone. When both paths hold data
|
||
# it moves neither and the store starts on the plain one.
|
||
systemd.services.victoriametrics-state-migrate =
|
||
let
|
||
old = "/var/lib/private/${config.services.victoriametrics.stateDir}";
|
||
new = "/var/lib/${config.services.victoriametrics.stateDir}";
|
||
in
|
||
{
|
||
description = "move VictoriaMetrics' DynamicUser state directory to its static path";
|
||
before = [ "victoriametrics.service" ];
|
||
requiredBy = [ "victoriametrics.service" ];
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
SyslogIdentifier = "victoriametrics-state-migrate";
|
||
};
|
||
path = [ pkgs.coreutils ];
|
||
script = ''
|
||
set -eu
|
||
old=${lib.escapeShellArg old}
|
||
new=${lib.escapeShellArg new}
|
||
if [ ! -d "$old" ] || [ -L "$old" ]; then
|
||
exit 0
|
||
fi
|
||
if [ -L "$new" ]; then
|
||
rm "$new"
|
||
elif [ -d "$new" ]; then
|
||
if [ -n "$(ls -A "$new")" ]; then
|
||
echo "victoriametrics-state-migrate: $old and $new both hold data; moving neither" >&2
|
||
exit 0
|
||
fi
|
||
rmdir "$new"
|
||
fi
|
||
mv "$old" "$new"
|
||
chown -R victoriametrics:gateway-nginx "$new"
|
||
echo "victoriametrics-state-migrate: moved $old to $new"
|
||
'';
|
||
};
|
||
|
||
services.victoriametrics = {
|
||
enable = true;
|
||
package = deployCfg.victoriametrics.package;
|
||
retentionPeriod = deployCfg.victoriametrics.retentionPeriod;
|
||
|
||
# ⚠️ A socket, not upstream's default `:8428` on every interface:
|
||
# the gateway is the only intended client, and this listener
|
||
# carries an unauthenticated write endpoint (see the OTLP note
|
||
# below).
|
||
listenAddress = "unix:${socketPath}";
|
||
};
|
||
};
|
||
};
|
||
};
|
||
|
||
# 🔑 OTLP ingest needs no flag. Measured against the pinned 1.146.0 rather
|
||
# than inferred from the module's option list, which has no OTLP switch and
|
||
# so reads as though the feature were absent: the running server answers
|
||
# `POST /opentelemetry/api/v1/push` with 200 (a nonexistent path answers
|
||
# 400, so that 200 means the route exists). The `-opentelemetry.*` flags
|
||
# only tune naming and limits, and are reachable via `extraOptions` if a
|
||
# deployment ever needs them.
|
||
#
|
||
# ⚠️ That endpoint is unauthenticated, which is why `listenAddress` above is
|
||
# a socket only the gateway can reach: the only route to it is the vhost's
|
||
# ingest location, which authenticates. The collector is still the only writer, but
|
||
# it now arrives by the swarm name rather than over loopback, because it need
|
||
# not share a host with this store.
|
||
}
|