vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-otel: scrape VictoriaMetrics through its published name

Browse source 
The store's own /metrics was scraped over loopback, declared under
deploy.victoriametrics.enable, so a collector on another host never had
a job for it. Declare it as a published target on every host instead,
keyed on swarm.victoriametrics.domain, and serve it on the metrics vhost
behind the existing authz subrequest. Authelia's per-target rules (one
collector rule, then deny) now cover it too.

The loopback entry goes, so a single-host swarm scrapes the store once.
This commit is contained in:
atlas 2026-10-13 10:44:50 +02:00 • committed by mara
commit 338a767a61
4 changed files with 80 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

5
docs/swarm/services.md
View file

@ -172,8 +172,9 @@ Neither container is reachable except through the gateway: both bind
loopback, and VictoriaMetrics authenticates nothing itself. The
`metrics.<swarm-domain>` vhost forwards only the store's read API (the
Prometheus query, series, label and metadata routes, plus the vmui web UI)
without a credential. It forwards the collector's OTLP push only with an
authelia token, and refuses every other path, the store's write and admin
without a credential. It forwards the collector's OTLP push, and the
collector's scrape of the store's own `/metrics`, only with an authelia
token, and refuses every other path, the store's write and admin
API included, with 403. The bare domain redirects to vmui.
### Logs (VictoriaLogs)

2
nix/host-modules/swarm-otel-service.nix
View file

@ -358,5 +358,7 @@ in
forgejo = "https://${hyperhiveCfg.swarm.forge.domain}/metrics";
# The vhost location is in ./swarm-authelia.nix.
authelia = "https://${hyperhiveCfg.swarm.authelia.domain}/metrics";
# The vhost location is in ./swarm-victoriametrics.nix.
victoriametrics = "https://${vmCfg.domain}/metrics";
};
}

39
nix/host-modules/swarm-victoriametrics.nix
View file

@ -116,22 +116,6 @@ in
services.hyperhive.gateway.enable = lib.mkDefault true;
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
# This store publishes its own health as prometheus metrics on the same
# listener it serves queries on, so the swarm's collector can scrape it
# with no exporter and no extra port.
#
# Declared here rather than in the collector's module because that is the
# rule the option carries: an entry exists only where the service that
# named it runs, which is what keeps scraper and target on one host by
# construction instead of by luck.
#
# The loopback literal is safe for this option and NOT for the write path,
# which is the distinction that matters now that the two differ. A scrape
# target is only ever read by a collector on this host, so loopback states
# a fact. The collector's push goes to the swarm name through the gateway,
# because the collector need not be here at all.
services.hyperhive.swarm.otel.scrapeTargets.victoriametrics = "127.0.0.1:${toString cfg.port}";
services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // {
listen = gatewayCfg.lib.listen;
extraConfig = gatewayCfg.lib.securityHeaders;
@ -146,8 +130,8 @@ in
# index page links mostly to paths refused above, so send it to vmui.
"= /".return = "302 /vmui/";
# The swarm collector's ingest route, and the only authenticated thing
# on this vhost. `=` so it outranks the `/` prefix above.
# The swarm collector's ingest route. `=` so it outranks the `/`
# prefix above.
#
# ⚠️ No `error_page 401 =302` here, and its absence is the point: a
# redirect is right for a browser and wrong for a pusher, which would
@ -160,7 +144,24 @@ in
'';
};
# The subrequest. Same target and header set as the sibling log
# The store's own health, for the swarm collector wherever it runs:
# served on the query listener, so no exporter and no extra port. `=`
# for the same reason as the push route, and no `error_page` for the
# same reason: a scraper would parse a login page as metrics.
#
# The collector requests exactly this URL, declared as
# `publishedScrapeTargets.victoriametrics` in ./swarm-otel-service.nix;
# the two must agree, since that string is also the token's audience.
"= /metrics" = {
proxyPass = "http://127.0.0.1:${toString cfg.port}/metrics";
extraConfig = ''
auth_request /__metrics_push_authz;
'';
};
# The subrequest for both gated locations above. It carries the
# requested URL, and authelia decides per URL, so the scrape needs no
# subrequest of its own. Same target and header set as the sibling log
# store's, which took them from `swarm-ui.nix` — `X-Original-URL` and
# `X-Original-Method` are what authelia's auth-request implementation
# reads, and the address it compares the token's audience against.

55
nix/module-eval/swarm-otel-identity.nix
View file

@ -106,6 +106,7 @@ let
forgeMetricsUrl = "https://forge.t.local/metrics";
autheliaMetricsUrl = "https://auth.t.local/metrics";
vmMetricsUrl = "https://metrics.t.local/metrics";
# The policy authelia applies to `subject` requesting `https://<domain><path>`,
# where `path` is what authelia matches `resources` against: the cleaned path
@ -251,6 +252,42 @@ let
&& authz.proxyPass == "https://${h.swarm.authelia.domain}/api/authz/auth-request"
&& lib.hasInfix "internal;" authz.extraConfig;
}
{
# The store's half of the same move, on a collector host that does not
# run the store.
name = "a collector on a host without the metrics store scrapes the store by name";
ok =
let
s = otelSettings otelRemoteAuthelia;
job = lib.findFirst (
j: j.job_name == "victoriametrics"
) null s.receivers.prometheus.config.scrape_configs;
in
!otelRemoteAuthelia.services.hyperhive.deploy.victoriametrics.enable
&& job != null
&& job.static_configs == [ { targets = [ "metrics.t.local" ]; } ]
&& job.metrics_path == "/metrics"
&& job.oauth2.endpoint_params.audience == vmMetricsUrl;
}
{
# The metrics vhost is default-deny; `/metrics` reaches the store only
# through its own exact location, gated like the push route. The `/`
# clauses pin that the catch-all still refuses every other path.
name = "the metrics store's vhost serves /metrics only behind the collector's auth subrequest";
ok =
let
h = allLocal.services.hyperhive;
locs = allLocal.services.nginx.virtualHosts.${h.swarm.victoriametrics.domain}.locations;
m = locs."= /metrics" or null;
in
m != null
&& m.proxyPass == "http://127.0.0.1:${toString h.swarm.victoriametrics.port}/metrics"
&& lib.hasInfix "auth_request /__metrics_push_authz;" m.extraConfig
&& !(lib.hasInfix "error_page" m.extraConfig)
&& lib.hasInfix "internal;" locs."= /__metrics_push_authz".extraConfig
&& locs."/".return == "403"
&& locs."/".proxyPass == null;
}
{
# Every published target admits the collector's client and refuses any
# other authenticated subject, with or without a query string. Read
@ -283,6 +320,7 @@ let
lib.sort lib.lessThan (lib.attrValues h.swarm.otel.publishedScrapeTargets) == [
autheliaMetricsUrl
forgeMetricsUrl
vmMetricsUrl
]
&& lib.length targets == 4
&& lib.all (t: policy collector t == "one_factor") targets
@ -367,6 +405,23 @@ let
&& lib.length jobs == 1
&& (lib.head jobs).static_configs == [ { targets = [ "auth.t.local" ]; } ];
}
{
# Same for the metrics store: one job, by its published name.
name = "a host running the metrics store and the collector scrapes the store once, by name";
ok =
let
o = allLocal.services.hyperhive.swarm.otel;
jobs =
lib.filter (j: j.job_name == "victoriametrics")
(otelSettings allLocal).receivers.prometheus.config.scrape_configs;
in
allLocal.services.hyperhive.deploy.victoriametrics.enable
&& allLocal.services.hyperhive.deploy.swarm-otel.enable
&& !(o.scrapeTargets ? victoriametrics)
&& (o.publishedScrapeTargets.victoriametrics or null) == vmMetricsUrl
&& lib.length jobs == 1
&& (lib.head jobs).static_configs == [ { targets = [ "metrics.t.local" ]; } ];
}
{
# The case above reads the options; this one reads the assertion they
# feed, which is what refuses an operator's `nixos-rebuild`.