223cca2b56
No access rule covered the push route on `metrics.<swarm>`, so a request there fell to `default_policy = one_factor`: any logged-in session cookie for the swarm could write samples into VictoriaMetrics. The name now gets two rules with no `resources`: the collector's client at `one_factor`, then `deny`. Only the push route and `/metrics` on that vhost send an authelia subrequest (the read API, Grafana's datasource included, is served with no check), and the collector is the one caller of both, so a domain-wide pair covers them with no path or query regex to slip past. It replaces the per-path `/metrics` pair the published target would otherwise derive for that name. Both rules sit ahead of the apex and store-UI rules, and authelia matches `domain` exactly, so they shadow no other name. Authelia 4.39.20, read from source: a rule with no `domain` wildcard matches with `strings.EqualFold` (access_control_domain.go), an empty `resources` or `subject` list matches everything (access_control_rule.go), and the first matching rule wins (authorizer.go). Fixes #5218
242 lines
10 KiB
Nix
242 lines
10 KiB
Nix
# The swarm's metrics store: one VictoriaMetrics for the whole swarm, in a
|
|
# `swarm-victoriametrics` nixos-container.
|
|
#
|
|
# A LOCAL time-series database rather than only an external sink, and that is
|
|
# the point rather than a convenience: the swarm dashboard has to stay
|
|
# readable when the outside world is unreachable. Same failure-domain property
|
|
# the hive-status KV has — a view of the system must not depend on the system
|
|
# it is viewing being healthy.
|
|
#
|
|
# It is the collector that feeds this (the gateway OTEL collector), not the
|
|
# agents directly: one ingest point per swarm, authenticated there.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.swarm.victoriametrics;
|
|
deployCfg = config.services.hyperhive.deploy;
|
|
networkCfg = config.services.hyperhive.network;
|
|
hyperhiveCfg = config.services.hyperhive;
|
|
gatewayCfg = hyperhiveCfg.gateway;
|
|
|
|
# Shared host netns, like every sibling swarm container: the gateway
|
|
# reaches this at 127.0.0.1:<port>.
|
|
privateNetwork = false;
|
|
|
|
# The store's read API: the only part of it the vhost serves without a
|
|
# credential. Each location's `proxy_pass` carries the path, so the store
|
|
# receives the path nginx matched, not the caller's unnormalised spelling.
|
|
#
|
|
# Grafana's Prometheus datasource reads the query, series, label, metadata,
|
|
# buildinfo, exemplar, rules and alerts routes; the store answers the last
|
|
# three with fixed placeholders. vmui is the static files under `/vmui/`
|
|
# plus the query API and the remaining read-only routes here. Paths read off
|
|
# the pinned build's `app/vmselect/main.go` and vmui's `src/`.
|
|
readPaths = [
|
|
"/api/v1/query"
|
|
"/api/v1/query_range"
|
|
"/api/v1/query_exemplars"
|
|
"/api/v1/series"
|
|
"/api/v1/labels"
|
|
"/api/v1/metadata"
|
|
"/api/v1/rules"
|
|
"/api/v1/alerts"
|
|
"/api/v1/status/buildinfo"
|
|
"/api/v1/export"
|
|
"/api/v1/export/csv"
|
|
"/api/v1/status/tsdb"
|
|
"/api/v1/status/metric_names_stats"
|
|
"/api/v1/status/top_queries"
|
|
"/api/v1/status/active_queries"
|
|
"/expand-with-exprs"
|
|
"/prettify-query"
|
|
"/metric-relabel-debug"
|
|
"/vmui"
|
|
];
|
|
# The store serves only `<name>/values` under `/api/v1/label/`, and only
|
|
# static files and UI config under `/vmui/`.
|
|
readPrefixes = [
|
|
"/api/v1/label/"
|
|
"/vmui/"
|
|
];
|
|
readLocations =
|
|
let
|
|
to = path: { proxyPass = "http://127.0.0.1:${toString cfg.port}${path}"; };
|
|
in
|
|
lib.listToAttrs (
|
|
map (p: lib.nameValuePair "= ${p}" (to p)) readPaths
|
|
++ map (p: lib.nameValuePair "^~ ${p}" (to p)) readPrefixes
|
|
);
|
|
in
|
|
{
|
|
# What the store IS from any hive's point of view, the name it answers on and
|
|
# the port, is `swarm.victoriametrics` in ./swarm-victoriametrics-service.nix.
|
|
# `enable`, `package` and `retentionPeriod` are decisions of the host that
|
|
# runs it and live under `deploy.*`.
|
|
|
|
# Retention is a property of the store this host runs, not something the
|
|
# swarm has to agree on: it is read only where the container is defined,
|
|
# and a hive that is a *client* of the metrics store never consults it.
|
|
options.services.hyperhive.deploy.victoriametrics.package = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = pkgs.victoriametrics;
|
|
defaultText = lib.literalExpression "pkgs.victoriametrics";
|
|
description = "VictoriaMetrics package to run.";
|
|
};
|
|
|
|
options.services.hyperhive.deploy.victoriametrics.retentionPeriod = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "5y";
|
|
example = "90d";
|
|
description = ''
|
|
How long samples are kept.
|
|
|
|
Deliberately a high default rather than a required option: the two
|
|
failure directions are not symmetric. Too long fills a disk, which
|
|
is visible and recoverable by lowering this; too short **destroys
|
|
history**, silently and permanently. So the safe default is generous
|
|
and an operator lowers it once they have measured how fast this swarm
|
|
actually accumulates data.
|
|
'';
|
|
};
|
|
|
|
config = lib.mkIf deployCfg.victoriametrics.enable {
|
|
# The gateway name, inside `deployCfg.victoriametrics.enable` — that
|
|
# guard is the load-bearing part. Every hive in a swarm may know this
|
|
# store exists, but only the host that RUNS it may claim the name; a
|
|
# client hive declaring the vhost would answer for a service it does not
|
|
# have.
|
|
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
|
|
|
# This host serves the vhost, and the container behind it resolves
|
|
# through the hive's dnsmasq.
|
|
services.hyperhive.gateway.enable = lib.mkDefault true;
|
|
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
|
|
|
services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // {
|
|
listen = gatewayCfg.lib.listen;
|
|
extraConfig = gatewayCfg.lib.securityHeaders;
|
|
locations = readLocations // {
|
|
# Nothing reaches the store unless a location here names it. The
|
|
# store serves writes and admin calls (`/api/v1/import`,
|
|
# `/api/v1/admin/tsdb/delete_series`, ...) on the same listener as
|
|
# its queries, and authenticates none of them.
|
|
"/".return = "403";
|
|
|
|
# The swarm UI's quick link opens the bare domain. The store's own
|
|
# index page links mostly to paths refused above, so send it to vmui.
|
|
"= /".return = "302 /vmui/";
|
|
|
|
# The swarm collector's ingest route. `=` so it outranks the `/`
|
|
# prefix above.
|
|
#
|
|
# ⚠️ No `error_page 401 =302` here, and its absence is the point: a
|
|
# redirect is right for a browser and wrong for a pusher, which would
|
|
# follow it and POST its batch at a login page that answers 200 —
|
|
# ingest reporting healthy while storing nothing.
|
|
"= /opentelemetry/api/v1/push" = {
|
|
proxyPass = "http://127.0.0.1:${toString cfg.port}/opentelemetry/api/v1/push";
|
|
extraConfig = ''
|
|
auth_request /__metrics_push_authz;
|
|
'';
|
|
};
|
|
|
|
# The store's own health, for the swarm collector wherever it runs:
|
|
# served on the query listener, so no exporter and no extra port. `=`
|
|
# for the same reason as the push route, and no `error_page` for the
|
|
# same reason: a scraper would parse a login page as metrics.
|
|
#
|
|
# The collector requests exactly this URL, declared as
|
|
# `publishedScrapeTargets.victoriametrics` in ./swarm-otel-service.nix;
|
|
# the two must agree, since that string is also the token's audience.
|
|
"= /metrics" = {
|
|
proxyPass = "http://127.0.0.1:${toString cfg.port}/metrics";
|
|
extraConfig = ''
|
|
auth_request /__metrics_push_authz;
|
|
'';
|
|
};
|
|
|
|
# The subrequest for both gated locations above. Authelia admits the
|
|
# collector's client anywhere on this name and refuses every other
|
|
# subject (`metricsStoreRules` in ./swarm-authelia.nix), so the scrape
|
|
# needs no subrequest of its own. Same target and header set as the
|
|
# sibling log store's, which took them from `swarm-ui.nix` —
|
|
# `X-Original-URL` and `X-Original-Method` are what authelia's
|
|
# auth-request implementation reads, and the address it compares the
|
|
# token's audience against.
|
|
"= /__metrics_push_authz" = {
|
|
proxyPass = "https://${hyperhiveCfg.swarm.authelia.domain}/api/authz/auth-request";
|
|
# nixpkgs appends its OWN `Host $host` after extraConfig, which
|
|
# would override verifiedProxyTo's — see the comment on
|
|
# verifiedProxyTo in hive-gateway/vhost-lib.nix.
|
|
recommendedProxySettings = false;
|
|
extraConfig = ''
|
|
internal;
|
|
${gatewayCfg.lib.verifiedProxyTo hyperhiveCfg.swarm.authelia.domain}
|
|
proxy_pass_request_body off;
|
|
proxy_set_header Content-Length "";
|
|
proxy_set_header X-Original-Method $request_method;
|
|
proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
|
|
proxy_set_header X-Forwarded-Proto $scheme;
|
|
proxy_set_header X-Forwarded-Host $http_host;
|
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
|
'';
|
|
};
|
|
};
|
|
};
|
|
|
|
containers.${cfg.machine} = {
|
|
autoStart = true;
|
|
ephemeral = false;
|
|
# Journal files on the host, not inside the container: nixpkgs hardcodes
|
|
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
|
|
extraFlags = [ "--link-journal=host" ];
|
|
inherit privateNetwork;
|
|
|
|
config =
|
|
{ ... }:
|
|
{
|
|
imports = [
|
|
../container-modules/swarm-container.nix
|
|
(import ../container-modules/swarm-container-resolver.nix {
|
|
inherit (networkCfg) bridgeIp;
|
|
dnsConsumers = [ "victoriametrics.service" ];
|
|
})
|
|
];
|
|
|
|
services.hyperhive.swarmContainer = { inherit privateNetwork; };
|
|
|
|
services.victoriametrics = {
|
|
enable = true;
|
|
package = deployCfg.victoriametrics.package;
|
|
retentionPeriod = deployCfg.victoriametrics.retentionPeriod;
|
|
|
|
# ⚠️ PINNED TO LOOPBACK, and this is a correction rather than a
|
|
# preference: upstream's default is `:8428`, i.e. every
|
|
# interface. The gateway is the only intended client and it is on
|
|
# this host, so binding wider would publish an unauthenticated
|
|
# write endpoint (see the OTLP note below) to whatever the host
|
|
# is reachable on.
|
|
listenAddress = "127.0.0.1:${toString cfg.port}";
|
|
};
|
|
};
|
|
};
|
|
};
|
|
|
|
# 🔑 OTLP ingest needs no flag. Measured against the pinned 1.146.0 rather
|
|
# than inferred from the module's option list, which has no OTLP switch and
|
|
# so reads as though the feature were absent: the running server answers
|
|
# `POST /opentelemetry/api/v1/push` with 200 (a nonexistent path answers
|
|
# 400, so that 200 means the route exists). The `-opentelemetry.*` flags
|
|
# only tune naming and limits, and are reachable via `extraOptions` if a
|
|
# deployment ever needs them.
|
|
#
|
|
# ⚠️ That endpoint is unauthenticated, which is why `listenAddress` above is
|
|
# loopback: the only route to it from off-host is the vhost's ingest
|
|
# location, which authenticates. The collector is still the only writer, but
|
|
# it now arrives by the swarm name rather than over loopback, because it need
|
|
# not share a host with this store.
|
|
}
|