vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm metrics store: the read API needs Grafana's bearer or an operator session

Browse source 
The metrics vhost forwarded the store's read API (query, series, labels,
export, vmui, ...) with no credential, so anything that could reach
metrics.<swarm> could read every hive's and agent's metrics. mara ruled on
checked by an auth_request responder, and operators use their authelia
session.

- swarm-metrics-auth (new crate): auth_request responder on the metrics
  store's host. Compares the presented bearer, in constant time, with the
  one stored at swarm/services/<grafana client id>/metrics/bearer, read
  under its own bao leaf and cached for 60s. Static user, the post-#5264
  swarm-logs-auth unit shape.
- swarm-bao-grafana-metrics-bearer: on Grafana's host, creates that bearer
  in bao if absent (create without update, Grafana's existing leaf) and
  writes "Bearer <value>" into the container; the datasource sends it as
  its Authorization header via $__file{}.
- The vhost: read locations take auth_request through a map (a bearer goes
  to swarm-metrics-auth, anything else to authelia, no login redirect);
  vmui takes an authelia session with the login redirect; forceSSL like
  the logs vhost. The authelia subrequest location is renamed from
  /__metrics_push_authz to /__metrics_authelia since it now serves reads.
- Authelia: the collector keeps every path; everyone else is refused the
  collector's push and /metrics paths; group admins is admitted to the
  rest; then deny.
- bao: a swarm-metrics-auth cert role and policy (read on the one leaf),
  its leaf minted by glue-bao-tls, the CN reserved as a hive name, and
  Grafana's grant gains create on the bearer leaf.

Fixes #3596
This commit is contained in:
atlas 2026-10-14 19:53:36 +02:00 • committed by mara
commit 6aab2ebb65
24 changed files with 1126 additions and 200 deletions
Download patch file Download diff file Expand all files Collapse all files

1
nix/host-modules/default.nix
View file

@ -38,6 +38,7 @@
./glue-hive-otel-authelia.nix
./glue-hive-tls-bao-identity.nix
./glue-logs-auth-bao-identity.nix
./glue-metrics-auth-bao-identity.nix
./glue-matrix-ctl-bao-identity.nix
./glue-matrix-oidc-client.nix
./glue-nats-auth-bao-identity.nix

6
nix/host-modules/glue-bao-readers-policy-order.nix
View file

@ -67,11 +67,15 @@ let
];
};
# ./swarm-grafana.nix, under Grafana's leaf, which may create the admin
# password only once that policy is written.
# password and the metrics bearer only once that policy is written.
swarm-bao-grafana-admin = {
here = deployCfg.grafana.enable;
policies = [ "swarm-bao-grafana-oidc-policy" ];
};
swarm-bao-grafana-metrics-bearer = {
here = deployCfg.grafana.enable;
policies = [ "swarm-bao-grafana-oidc-policy" ];
};
# ./hive-matrix.nix, under matrix-ctl's leaf.
swarm-bao-matrix-oidc = {
here = deployCfg.matrix.enable;

1
nix/host-modules/glue-bao-readers-store-order.nix
View file

@ -46,6 +46,7 @@ let
# ./swarm-grafana.nix
swarm-bao-grafana-oidc = deployCfg.grafana.enable;
swarm-bao-grafana-admin = deployCfg.grafana.enable;
swarm-bao-grafana-metrics-bearer = deployCfg.grafana.enable;
# ./swarm-nats.nix
swarm-bao-nats-callout-xkey = deployCfg.nats.enable;
# ./swarm-ci-runner.nix

7
nix/host-modules/glue-bao-tls.nix
View file

@ -276,6 +276,13 @@ in
[ -s ${pkiDir}/logs-auth.pem ] || ${signLeaf} ${pkiDir} logs-auth \
${lib.escapeShellArg deployCfg.bao.logsAuthCommonName} "" clientAuth
# The metrics store's Grafana auth responder, which reads Grafana's
# metrics bearer. Minted here for the matrix-ctl leaf's reason: the
# metrics store is a swarm singleton, so elsewhere this is the file an
# operator copies.
[ -s ${pkiDir}/metrics-auth.pem ] || ${signLeaf} ${pkiDir} metrics-auth \
${lib.escapeShellArg deployCfg.bao.metricsAuthCommonName} "" clientAuth
# The CI runner's token fetch. Minted here for the matrix-ctl leaf's
# reason: the runner is a swarm singleton, so on a runner host without
# the store this is the file an operator copies.

32
nix/host-modules/glue-metrics-auth-bao-identity.nix Normal file
View file

@ -0,0 +1,32 @@
# Glue: point the metrics store's Grafana auth responder at the bao leaf minted
# for it.
#
# ONE PAIRING PER FILE — the swarm-metrics-auth principal ← bao, and nothing
# else. Deleting this leaves a metrics store with no responder identity unless
# the operator names one, which ./swarm-victoriametrics.nix refuses at eval.
#
# ⚠️ The minting is NOT here. ./glue-bao-tls.nix holds the CA and signs the
# leaf. What belongs here is the pairing: which paths the responder presents.
#
# ⚠️ Gated on the store running on this host, the only place the leaf is
# minted. A metrics-store host without the store gets no default here; the
# operator places the responder's leaf and names it.
#
# Everything is `mkDefault`. An operator naming their own paths wins.
{
lib,
config,
...
}:
let
deployCfg = config.services.hyperhive.deploy;
pkiDir = import ./lib/bao-pki-dir.nix;
in
{
config = lib.mkIf (deployCfg.victoriametrics.enable && deployCfg.bao.enable) {
services.hyperhive.deploy.victoriametrics = {
authBaoClientCertFile = lib.mkDefault "${pkiDir}/metrics-auth.pem";
authBaoClientKeyFile = lib.mkDefault "${pkiDir}/metrics-auth-key.pem";
};
};
}

43
nix/host-modules/swarm-authelia.nix
View file

@ -97,14 +97,43 @@ let
)
++ [ (target // { policy = "deny"; }) ];
# The metrics store's name, every path. Only its push route and `/metrics`
# consult authelia (its read API is served with no check), and the
# collector is the one caller of both, so the rules take no `resources`
# and leave no path or query form to fall through to `default_policy`.
# ⚠️ An authenticated location added to that vhost for anyone else is
# refused here until it gets a rule ahead of the `deny`.
# The metrics store's name, every path. Authelia is consulted there for the
# collector's push route and `/metrics`, and for an operator's session on
# the read API and vmui; Grafana's bearer on the read API is checked by
# `swarm-metrics-auth`, never here.
#
# The collector's client is admitted anywhere on the name. Every other
# subject is refused the collector's two paths, then operators are admitted
# to the rest, then everything else is refused. Prefixes rather than exact
# paths for the collector's two, so no spelling of them reaches the
# operator rule. No rule takes a narrower `resources` than that, so no path
# or query form falls through to `default_policy`.
metricsStoreDomain = config.services.hyperhive.swarm.victoriametrics.domain;
metricsStoreRules = collectorOnly { domain = metricsStoreDomain; };
metricsStoreRules =
lib.optional collectorRegistered {
domain = metricsStoreDomain;
policy = "one_factor";
subject = [ "oauth2:client:${collectorClientId}" ];
}
++ [
{
domain = metricsStoreDomain;
policy = "deny";
resources = [
"^/opentelemetry/.*"
"^/metrics.*"
];
}
{
domain = metricsStoreDomain;
policy = "one_factor";
subject = [ "group:${operatorGroup}" ];
}
{
domain = metricsStoreDomain;
policy = "deny";
}
];
# One pair per `swarm.otel.publishedScrapeTargets` entry, read off the same
# URL the collector requests and its token is minted for, so a target

40
nix/host-modules/swarm-bao.nix
View file

@ -908,15 +908,17 @@ let
# the objects they write.
#
# Grafana's leaf also creates its admin password (`swarm-bao-grafana-admin`
# in ./swarm-grafana.nix). `create` without `update`, so a stored password is
# never replaced.
# in ./swarm-grafana.nix) and the bearer its metrics datasource presents
# (`swarm-bao-grafana-metrics-bearer`). `create` without `update`, so a stored
# value is never replaced.
grafanaOidcReaders = [
{
name = "swarm-grafana-oidc";
cn = baoDeploy.grafanaOidcCommonName;
policyText =
readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/oidc/client"
+ createStanza "${credentialMountPath}/data/${grafanaAdminLeaf}";
+ createStanza "${credentialMountPath}/data/${grafanaAdminLeaf}"
+ createStanza "${credentialMountPath}/data/${grafanaMetricsBearerLeaf}";
}
];
@ -982,6 +984,12 @@ let
# of `adminPasswordStorePath` in ./swarm-grafana.nix.
grafanaAdminLeaf = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/admin/password";
# The bearer Grafana's metrics datasource presents at the metrics store.
# Grafana's leaf creates it, the store's responder reads it. The nix half of
# `metricsBearerStorePath` in ./swarm-grafana.nix and of `--store-path` in
# ./swarm-victoriametrics.nix.
grafanaMetricsBearerLeaf = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer";
createStanza = path: ''
path "${path}" {
capabilities = ["create", "read"]
@ -999,6 +1007,16 @@ let
}
];
# The metrics store's responder, which checks the bearer Grafana presents
# against the one stored for it, and reads nothing else.
metricsAuthReaders = [
{
name = "swarm-metrics-auth";
cn = baoDeploy.metricsAuthCommonName;
policyText = readStanza "${credentialMountPath}/data/${grafanaMetricsBearerLeaf}";
}
];
# The CI runner's token fetch. The one reader of the path, which the
# controller writes and cannot read back.
ciRunnerReaders = [
@ -1859,6 +1877,19 @@ in
'';
};
metricsAuthCommonName = lib.mkOption {
type = lib.types.str;
default = "swarm-metrics-auth";
description = ''
Subject the store's `swarm-metrics-auth` cert-auth role accepts: the
identity the metrics store's Grafana auth responder presents to read
the bearer Grafana's datasource sends,
`swarm/services/<grafana client id>/metrics/bearer`, and nothing else.
⚠️ Reserved as a hive name by ./swarm.nix, like its siblings.
'';
};
ciRunnerCommonName = lib.mkOption {
type = lib.types.str;
default = "swarm-ci-runner";
@ -3274,6 +3305,8 @@ in
systemd.services.swarm-bao-logs-auth-policy = readerPolicyUnit "write the log store responder's agent-credential-reader bao policy and cert-auth role" logsAuthReaders;
systemd.services.swarm-bao-metrics-auth-policy = readerPolicyUnit "write the metrics store responder's Grafana-bearer-reader bao policy and cert-auth role" metricsAuthReaders;
systemd.services.swarm-bao-ci-runner-policy = readerPolicyUnit "write the CI runner's registration-token-reader bao policy and cert-auth role" ciRunnerReaders;
systemd.services.swarm-bao-review-classifier-policy = readerPolicyUnit "write the review classifier's API-key-reader bao policy and cert-auth role" reviewClassifierReaders;
@ -3408,6 +3441,7 @@ in
"swarm-bao-hive-leaf-policy.service"
"swarm-bao-ci-runner-policy.service"
"swarm-bao-logs-auth-policy.service"
"swarm-bao-metrics-auth-policy.service"
"swarm-bao-forge-publisher-policy.service"
]
++ granterAfter;

103
nix/host-modules/swarm-grafana.nix
View file

@ -263,6 +263,16 @@ let
adminPasswordPath = "/var/lib/grafana-admin/password";
hostAdminPasswordPath = "/var/lib/nixos-containers/${cfg.machine}${adminPasswordPath}";
# The bearer the metrics datasource presents at the metrics store's query
# API, which `swarm-metrics-auth` on the store's host checks against the
# same store path. Minted once by `swarm-bao-grafana-metrics-bearer` below,
# under the same grant as the admin password. The file holds the whole
# header value, `Bearer <value>`: nixpkgs accepts a `secureJsonData` entry
# only when it is exactly one `$__file{}`.
metricsBearerStorePath = "secret/swarm/services/${cfg.oidc.clientId}/metrics/bearer";
metricsBearerPath = "/var/lib/grafana-metrics/bearer";
hostMetricsBearerPath = "/var/lib/nixos-containers/${cfg.machine}${metricsBearerPath}";
socketPath = "${deployCfg.grafana.socketDir}/grafana.sock";
# Both static NixOS ids, and both checked rather than assumed: nginx has
@ -328,9 +338,12 @@ in
description = ''
Where the provisioned datasource points. Defaults to the metrics
store's swarm name, served by the gateway of whichever host runs the
store, so Grafana and the store need not share a host. The store's
query routes on that vhost take no credential, so the datasource
sends none.
store, so Grafana and the store need not share a host.
The datasource sends the bearer `swarm-bao-grafana-metrics-bearer`
keeps in the swarm secret store, which is what the store's query
routes on that vhost check. Whatever URL this names receives that
bearer.
'';
};
@ -756,6 +769,86 @@ in
'';
};
# The metrics datasource's bearer: created in the store if it holds none,
# then copied into the container, under Grafana's own leaf. Same shape and
# grant as the admin password above. The store's `swarm-metrics-auth`
# reads the same path, so nothing is placed by hand on either host.
systemd.services.swarm-bao-grafana-metrics-bearer = {
description = "create Grafana's metrics store bearer in the swarm secret store if absent and fetch it";
# Ordered after the store where it runs here:
# ./glue-bao-readers-store-order.nix.
before = [ "container@${cfg.machine}.service" ];
wantedBy = [
"multi-user.target"
"container@${cfg.machine}.service"
];
path = [
baoDeploy.package
pkgs.coreutils
pkgs.openssl
pkgs.systemd
];
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
RemainAfterExit = true;
SyslogIdentifier = "swarm-bao-grafana-metrics-bearer";
TimeoutStartSec = 30;
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = baoDeploy.grafanaOidcClientCertFile;
BAO_CLIENT_KEY = baoDeploy.grafanaOidcClientKeyFile;
}
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
BAO_CACERT = baoDeploy.serverCaFile;
};
script = ''
set -euo pipefail
${atomicWriteSecret}
${refreshConsumer}
err="$(mktemp)"
trap 'rm -f "$err"' EXIT
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to swarm-bao with Grafana's certificate; leaving Grafana's metrics bearer file as it is." >&2
cat "$err" >&2
exit 1
fi
export BAO_TOKEN
# Read, create on a miss, read again: the admin password's sequence
# above, for its reasons.
if ! bearer="$(bao kv get -field=value ${lib.escapeShellArg metricsBearerStorePath} 2>"$err")"; then
minted="$(openssl rand -hex 32)"
if ! printf '%s' "$minted" | bao kv put -cas=0 ${lib.escapeShellArg metricsBearerStorePath} value=- >/dev/null 2>"$err"; then
echo "could not create ${metricsBearerStorePath}; reading it back in case another run created it." >&2
cat "$err" >&2
fi
if ! bearer="$(bao kv get -field=value ${lib.escapeShellArg metricsBearerStorePath} 2>"$err")"; then
echo "swarm-bao did not return ${metricsBearerStorePath}; Grafana does not start without it." >&2
cat "$err" >&2
exit 1
fi
fi
if [ -z "$bearer" ]; then
echo "swarm-bao returned an empty ${metricsBearerStorePath}; leaving the file as it is." >&2
exit 1
fi
# Owner and mode as for the OIDC secret above, for its reasons.
install -d -m 0755 ${lib.escapeShellArg (builtins.dirOf hostMetricsBearerPath)}
if secret_differs ${lib.escapeShellArg hostMetricsBearerPath} "Bearer $bearer"; then
atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg hostMetricsBearerPath} "Bearer $bearer"
fi
# `$__file{}` in a provisioned datasource is read as Grafana starts.
refresh_consumer ${lib.escapeShellArg cfg.machine} grafana.service ${lib.escapeShellArg hostMetricsBearerPath}
'';
};
containers.${cfg.machine} = {
autoStart = true;
ephemeral = false;
@ -1017,6 +1110,10 @@ in
url = deployCfg.grafana.datasourceUrl;
access = "proxy";
isDefault = true;
# The store's query routes admit this bearer or an operator's
# session, and nothing else.
jsonData.httpHeaderName1 = "Authorization";
secureJsonData.httpHeaderValue1 = "$__file{${metricsBearerPath}}";
}
{
name = "VictoriaLogs";

358
nix/host-modules/swarm-victoriametrics.nix
View file

@ -36,13 +36,14 @@ let
# container, which knows it as `gateway-nginx`.
nginxGid = config.ids.gids.nginx;
# The store's read API: the only part of it the vhost serves without a
# credential. Each location's `proxy_pass` carries the path, so the store
# receives the path nginx matched, not the caller's unnormalised spelling.
# The store's read API, which the vhost forwards to a caller holding
# Grafana's bearer or an operator's session. Each location's `proxy_pass`
# carries the path, so the store receives the path nginx matched, not the
# caller's unnormalised spelling.
#
# Grafana's Prometheus datasource reads the query, series, label, metadata,
# buildinfo, exemplar, rules and alerts routes; the store answers the last
# three with fixed placeholders. vmui is the static files under `/vmui/`
# three with fixed placeholders. vmui (`uiLocations` below) is static files
# plus the query API and the remaining read-only routes here. Paths read off
# the pinned build's `app/vmselect/main.go` and vmui's `src/`.
readPaths = [
@ -64,22 +65,67 @@ let
"/expand-with-exprs"
"/prettify-query"
"/metric-relabel-debug"
"/vmui"
];
# The store serves only `<name>/values` under `/api/v1/label/`, and only
# static files and UI config under `/vmui/`.
readPrefixes = [
"/api/v1/label/"
"/vmui/"
];
# The store serves only `<name>/values` under `/api/v1/label/`.
readPrefixes = [ "/api/v1/label/" ];
# Bare `auth_request`, no login redirect: a machine caller handed
# `error_page 401 =302` follows it and reads authelia's login page as a 200
# holding no series. `/__metrics_read_auth` picks the check from the
# caller's `Authorization` (`appendHttpConfig` below).
readAuth = ''
auth_request /__metrics_read_auth;
'';
readLocations =
let
to = path: { proxyPass = upstream path; };
to = path: {
proxyPass = upstream path;
extraConfig = readAuth;
};
in
lib.listToAttrs (
map (p: lib.nameValuePair "= ${p}" (to p)) readPaths
++ map (p: lib.nameValuePair "^~ ${p}" (to p)) readPrefixes
);
# The same three lines as `swarmAuthRequest` in ./swarm-victorialogs.nix,
# for the same browser: an operator without a session is sent to log in.
swarmAuthRequest = ''
auth_request /__metrics_authelia;
auth_request_set $target_url $scheme://$http_host$request_uri;
error_page 401 =302 https://${hyperhiveCfg.swarm.authelia.domain}/?rd=$target_url;
'';
# vmui's static files and UI config. Only a browser loads these, and its
# queries then go to the read locations above with the same session cookie.
uiLocations = {
"= /vmui" = {
proxyPass = upstream "/vmui";
extraConfig = swarmAuthRequest;
};
"^~ /vmui/" = {
proxyPass = upstream "/vmui/";
extraConfig = swarmAuthRequest;
};
};
# The `auth_request` responder for Grafana's bearer (`swarm-metrics-auth`).
# Its socket's directory holds that socket alone and is traverse-only for
# others, the shape `swarm-logs-auth` in ./swarm-victorialogs.nix has.
metricsAuthUnit = "swarm-metrics-auth";
metricsAuthSocket = "/run/${metricsAuthUnit}/auth.sock";
metricsAuthCredential = id: "/run/credentials/${metricsAuthUnit}.service/${id}";
# Credential id → host path of the responder's store identity.
metricsAuthStoreFiles = {
"bao-client.pem" = deployCfg.victoriametrics.authBaoClientCertFile;
"bao-client-key.pem" = deployCfg.victoriametrics.authBaoClientKeyFile;
}
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
"bao-ca.pem" = deployCfg.bao.serverCaFile;
};
baoCfg = hyperhiveCfg.swarm.bao;
# The path Grafana's host creates the bearer at, under the KV mount. The
# same leaf as `grafanaMetricsBearerLeaf` in ./swarm-bao.nix.
grafanaBearerStorePath = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer";
in
{
# What the store IS from any hive's point of view, the name it answers on,
@ -113,6 +159,44 @@ in
'';
};
options.services.hyperhive.deploy.victoriametrics.authPackage = lib.mkOption {
type = lib.types.package;
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-metrics-auth";
description = ''
The `swarm-metrics-auth` package: the gateway's `auth_request` responder
that checks the bearer Grafana's metrics datasource presents on the
store's query API against the secret store.
'';
};
options.services.hyperhive.deploy.victoriametrics.authBaoClientCertFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "/var/lib/swarm-bao-pki/metrics-auth.pem";
description = ''
Client certificate `swarm-metrics-auth` presents to the swarm's secret
store to read Grafana's metrics bearer. Its subject must be
{option}`services.hyperhive.deploy.bao.metricsAuthCommonName`.
Required when `deploy.victoriametrics.enable` is set; unset fails at
eval. ./glue-metrics-auth-bao-identity.nix points it at the leaf
./glue-bao-tls.nix mints on a host that runs the store; on any other
metrics-store host it is a file the operator places.
A path, never a value.
'';
};
options.services.hyperhive.deploy.victoriametrics.authBaoClientKeyFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "/var/lib/swarm-bao-pki/metrics-auth-key.pem";
description = ''
Private key for {option}`services.hyperhive.deploy.victoriametrics.authBaoClientCertFile`.
A path, never a value.
'';
};
config = lib.mkIf deployCfg.victoriametrics.enable {
assertions = [
{
@ -128,6 +212,22 @@ in
cannot connect and every read, push and scrape fails with 502.
'';
}
{
assertion =
deployCfg.victoriametrics.authBaoClientCertFile != null
&& deployCfg.victoriametrics.authBaoClientKeyFile != null;
message = ''
services.hyperhive.deploy.victoriametrics.authBaoClientCertFile and
authBaoClientKeyFile must both be set.
swarm-metrics-auth reads Grafana's metrics bearer from the swarm's
secret store with this certificate. They default to the leaf the
store mints where services.hyperhive.deploy.bao.enable is set on
this host; elsewhere, issue a leaf whose CN is
services.hyperhive.deploy.bao.metricsAuthCommonName and name it in
both options.
'';
}
];
# The gateway name, inside `deployCfg.victoriametrics.enable` — that
@ -141,77 +241,118 @@ in
services.hyperhive.gateway.enable = lib.mkDefault true;
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // {
listen = gatewayCfg.lib.listen;
extraConfig = gatewayCfg.lib.securityHeaders;
locations = readLocations // {
# Nothing reaches the store unless a location here names it. The
# store serves writes and admin calls (`/api/v1/import`,
# `/api/v1/admin/tsdb/delete_series`, ...) on the same listener as
# its queries, and authenticates none of them.
"/".return = "403";
# Which check a read location's `auth_request` gets. `auth_request` takes a
# fixed URI, so it names `/__metrics_read_auth`, which rewrites to the
# target this map picks: a bearer goes to Grafana's check, anything else
# (an operator's session cookie) to authelia.
services.nginx.appendHttpConfig = ''
map $http_authorization $swarm_metrics_read_auth_target {
"~^Bearer " /__metrics_grafana_auth;
default /__metrics_authelia;
}
'';
# The swarm UI's quick link opens the bare domain. The store's own
# index page links mostly to paths refused above, so send it to vmui.
"= /".return = "302 /vmui/";
# `removeAttrs`/`forceSSL`: authelia answers a plain-http auth subrequest
# with 400, which `auth_request` cannot read as anything but a broken
# check. Same as ./swarm-victorialogs.nix's vhost.
services.nginx.virtualHosts."${cfg.domain}" =
(builtins.removeAttrs (gatewayCfg.lib.tlsFor cfg.domain) [ "addSSL" ])
// {
forceSSL = true;
listen = gatewayCfg.lib.listen;
extraConfig = gatewayCfg.lib.securityHeaders;
locations =
readLocations
// uiLocations
// {
# Nothing reaches the store unless a location here names it. The
# store serves writes and admin calls (`/api/v1/import`,
# `/api/v1/admin/tsdb/delete_series`, ...) on the same listener as
# its queries, and authenticates none of them.
"/".return = "403";
# The swarm collector's ingest route. `=` so it outranks the `/`
# prefix above.
#
# ⚠️ No `error_page 401 =302` here, and its absence is the point: a
# redirect is right for a browser and wrong for a pusher, which would
# follow it and POST its batch at a login page that answers 200 —
# ingest reporting healthy while storing nothing.
"= /opentelemetry/api/v1/push" = {
proxyPass = upstream "/opentelemetry/api/v1/push";
extraConfig = ''
auth_request /__metrics_push_authz;
'';
};
# The swarm UI's quick link opens the bare domain. The store's own
# index page links mostly to paths refused above, so send it to vmui.
"= /".return = "302 /vmui/";
# The store's own health, for the swarm collector wherever it runs:
# served on the query listener, so no exporter and no extra port. `=`
# for the same reason as the push route, and no `error_page` for the
# same reason: a scraper would parse a login page as metrics.
#
# The collector requests exactly this URL, declared as
# `publishedScrapeTargets.victoriametrics` in ./swarm-otel-service.nix;
# the two must agree, since that string is also the token's audience.
"= /metrics" = {
proxyPass = upstream "/metrics";
extraConfig = ''
auth_request /__metrics_push_authz;
'';
};
# The swarm collector's ingest route. `=` so it outranks the `/`
# prefix above.
#
# ⚠️ No `error_page 401 =302` here, and its absence is the point: a
# redirect is right for a browser and wrong for a pusher, which would
# follow it and POST its batch at a login page that answers 200 —
# ingest reporting healthy while storing nothing.
"= /opentelemetry/api/v1/push" = {
proxyPass = upstream "/opentelemetry/api/v1/push";
extraConfig = ''
auth_request /__metrics_authelia;
'';
};
# The subrequest for both gated locations above. Authelia admits the
# collector's client anywhere on this name and refuses every other
# subject (`metricsStoreRules` in ./swarm-authelia.nix), so the scrape
# needs no subrequest of its own. Same target and header set as the
# sibling log store's, which took them from `swarm-ui.nix` —
# `X-Original-URL` and `X-Original-Method` are what authelia's
# auth-request implementation reads, and the address it compares the
# token's audience against.
"= /__metrics_push_authz" = {
proxyPass = "https://${hyperhiveCfg.swarm.authelia.domain}/api/authz/auth-request";
# nixpkgs appends its OWN `Host $host` after extraConfig, which
# would override verifiedProxyTo's — see the comment on
# verifiedProxyTo in hive-gateway/vhost-lib.nix.
recommendedProxySettings = false;
extraConfig = ''
internal;
${gatewayCfg.lib.verifiedProxyTo hyperhiveCfg.swarm.authelia.domain}
proxy_pass_request_body off;
proxy_set_header Content-Length "";
proxy_set_header X-Original-Method $request_method;
proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
'';
};
# The store's own health, for the swarm collector wherever it runs:
# served on the query listener, so no exporter and no extra port. `=`
# for the same reason as the push route, and no `error_page` for the
# same reason: a scraper would parse a login page as metrics.
#
# The collector requests exactly this URL, declared as
# `publishedScrapeTargets.victoriametrics` in ./swarm-otel-service.nix;
# the two must agree, since that string is also the token's audience.
"= /metrics" = {
proxyPass = upstream "/metrics";
extraConfig = ''
auth_request /__metrics_authelia;
'';
};
# `rewrite … last` re-matches locations, so the subrequest lands in
# one of the two internal locations named by the map.
"= /__metrics_read_auth" = {
extraConfig = ''
internal;
rewrite ^ $swarm_metrics_read_auth_target last;
'';
};
# Grafana's bearer, checked by `swarm-metrics-auth`. The
# `Authorization` header rides the subrequest unchanged. No rate limit
# in front: the responder reads the store at most once per its cache
# TTL, whatever the bearers.
"= /__metrics_grafana_auth" = {
proxyPass = "http://unix:${metricsAuthSocket}:/";
extraConfig = ''
internal;
proxy_pass_request_body off;
proxy_set_header Content-Length "";
'';
};
# The authelia subrequest. For the collector's two locations above it
# admits only the collector's client; for the read and vmui locations
# it admits an operator's session (`metricsStoreRules` in
# ./swarm-authelia.nix). Same target and header set as the sibling log
# store's, which took them from `swarm-ui.nix` — `X-Original-URL` and
# `X-Original-Method` are what authelia's auth-request implementation
# reads, and the address it compares the token's audience against.
"= /__metrics_authelia" = {
proxyPass = "https://${hyperhiveCfg.swarm.authelia.domain}/api/authz/auth-request";
# nixpkgs appends its OWN `Host $host` after extraConfig, which
# would override verifiedProxyTo's — see the comment on
# verifiedProxyTo in hive-gateway/vhost-lib.nix.
recommendedProxySettings = false;
extraConfig = ''
internal;
${gatewayCfg.lib.verifiedProxyTo hyperhiveCfg.swarm.authelia.domain}
proxy_pass_request_body off;
proxy_set_header Content-Length "";
proxy_set_header X-Original-Method $request_method;
proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
'';
};
};
};
};
# The socket directory, created host-side before the container starts:
# nixos-container refuses to start when a bind source is missing.
@ -224,6 +365,67 @@ in
"d ${socketDir} 0770 root ${toString nginxGid} - -"
];
# Grafana's check on the read API. On the host rather than in the store's
# container: it needs nothing of the store, only nginx's reach to its
# socket and its own store identity.
users.users.${metricsAuthUnit} = {
isSystemUser = true;
group = metricsAuthUnit;
description = "swarm metrics store auth_request responder";
};
users.groups.${metricsAuthUnit} = { };
systemd.services.${metricsAuthUnit} = {
description = "swarm metrics store auth_request responder for Grafana's bearer";
wantedBy = [ "multi-user.target" ];
# Where the store's PKI is minted on this host, the leaf is one of its
# files. Ordering only: elsewhere the unit does not exist.
after = [
"network.target"
"swarm-bao-pki.service"
];
serviceConfig = {
ExecStart = lib.concatStringsSep " " [
"${deployCfg.victoriametrics.authPackage}/bin/swarm-metrics-auth"
"--socket ${metricsAuthSocket}"
# The role ./swarm-bao.nix writes on the store's `cert` mount.
"--store-cert-role swarm-metrics-auth"
"--store-path ${grafanaBearerStorePath}"
];
# Paths only: `argv` is world-readable via /proc/<pid>/cmdline.
LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") metricsAuthStoreFiles;
# ⚠️ A static user, never `DynamicUser`: with `RuntimeDirectoryPreserve`
# systemd puts a dynamic user's runtime directory under
# `/run/private/` (0700 root:root), which nginx cannot traverse.
User = metricsAuthUnit;
Group = metricsAuthUnit;
RuntimeDirectory = metricsAuthUnit;
# Traverse-only for others, so nginx reaches the socket path.
RuntimeDirectoryMode = "0751";
# The path never vanishes from under a running nginx; the responder
# unlinks a stale socket as it starts.
RuntimeDirectoryPreserve = "yes";
Restart = "on-failure";
RestartSec = "5s";
SyslogIdentifier = metricsAuthUnit;
ProtectSystem = "strict";
ProtectHome = true;
PrivateTmp = "disconnected";
PrivateDevices = true;
NoNewPrivileges = true;
RestrictSUIDSGID = true;
RemoveIPC = true;
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = metricsAuthCredential "bao-client.pem";
BAO_CLIENT_KEY = metricsAuthCredential "bao-client-key.pem";
}
// lib.optionalAttrs (metricsAuthStoreFiles ? "bao-ca.pem") {
BAO_CACERT = metricsAuthCredential "bao-ca.pem";
};
};
containers.${cfg.machine} = {
autoStart = true;
ephemeral = false;

1
nix/host-modules/swarm.nix
View file

@ -56,6 +56,7 @@ let
deployCfg.bao.natsAuthCommonName
deployCfg.bao.ciRunnerCommonName
deployCfg.bao.logsAuthCommonName
deployCfg.bao.metricsAuthCommonName
deployCfg.bao.reviewClassifierCommonName
deployCfg.bao.builderCommonName
deployCfg.bao.forgePublisherCommonName