swarm-otel: scrape VictoriaMetrics through its published name
The store's own /metrics was scraped over loopback, declared under deploy.victoriametrics.enable, so a collector on another host never had a job for it. Declare it as a published target on every host instead, keyed on swarm.victoriametrics.domain, and serve it on the metrics vhost behind the existing authz subrequest. Authelia's per-target rules (one collector rule, then deny) now cover it too. The loopback entry goes, so a single-host swarm scrapes the store once.
This commit is contained in:
atlas
mara
parent
6d42abb285
commit
338a767a61
4 changed files with 80 additions and 21 deletions
|
|
@ -358,5 +358,7 @@ in
|
|||
forgejo = "https://${hyperhiveCfg.swarm.forge.domain}/metrics";
|
||||
# The vhost location is in ./swarm-authelia.nix.
|
||||
authelia = "https://${hyperhiveCfg.swarm.authelia.domain}/metrics";
|
||||
# The vhost location is in ./swarm-victoriametrics.nix.
|
||||
victoriametrics = "https://${vmCfg.domain}/metrics";
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -116,22 +116,6 @@ in
|
|||
services.hyperhive.gateway.enable = lib.mkDefault true;
|
||||
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
||||
|
||||
# This store publishes its own health as prometheus metrics on the same
|
||||
# listener it serves queries on, so the swarm's collector can scrape it
|
||||
# with no exporter and no extra port.
|
||||
#
|
||||
# Declared here rather than in the collector's module because that is the
|
||||
# rule the option carries: an entry exists only where the service that
|
||||
# named it runs, which is what keeps scraper and target on one host by
|
||||
# construction instead of by luck.
|
||||
#
|
||||
# The loopback literal is safe for this option and NOT for the write path,
|
||||
# which is the distinction that matters now that the two differ. A scrape
|
||||
# target is only ever read by a collector on this host, so loopback states
|
||||
# a fact. The collector's push goes to the swarm name through the gateway,
|
||||
# because the collector need not be here at all.
|
||||
services.hyperhive.swarm.otel.scrapeTargets.victoriametrics = "127.0.0.1:${toString cfg.port}";
|
||||
|
||||
services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // {
|
||||
listen = gatewayCfg.lib.listen;
|
||||
extraConfig = gatewayCfg.lib.securityHeaders;
|
||||
|
|
@ -146,8 +130,8 @@ in
|
|||
# index page links mostly to paths refused above, so send it to vmui.
|
||||
"= /".return = "302 /vmui/";
|
||||
|
||||
# The swarm collector's ingest route, and the only authenticated thing
|
||||
# on this vhost. `=` so it outranks the `/` prefix above.
|
||||
# The swarm collector's ingest route. `=` so it outranks the `/`
|
||||
# prefix above.
|
||||
#
|
||||
# ⚠️ No `error_page 401 =302` here, and its absence is the point: a
|
||||
# redirect is right for a browser and wrong for a pusher, which would
|
||||
|
|
@ -160,7 +144,24 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
# The subrequest. Same target and header set as the sibling log
|
||||
# The store's own health, for the swarm collector wherever it runs:
|
||||
# served on the query listener, so no exporter and no extra port. `=`
|
||||
# for the same reason as the push route, and no `error_page` for the
|
||||
# same reason: a scraper would parse a login page as metrics.
|
||||
#
|
||||
# The collector requests exactly this URL, declared as
|
||||
# `publishedScrapeTargets.victoriametrics` in ./swarm-otel-service.nix;
|
||||
# the two must agree, since that string is also the token's audience.
|
||||
"= /metrics" = {
|
||||
proxyPass = "http://127.0.0.1:${toString cfg.port}/metrics";
|
||||
extraConfig = ''
|
||||
auth_request /__metrics_push_authz;
|
||||
'';
|
||||
};
|
||||
|
||||
# The subrequest for both gated locations above. It carries the
|
||||
# requested URL, and authelia decides per URL, so the scrape needs no
|
||||
# subrequest of its own. Same target and header set as the sibling log
|
||||
# store's, which took them from `swarm-ui.nix` — `X-Original-URL` and
|
||||
# `X-Original-Method` are what authelia's auth-request implementation
|
||||
# reads, and the address it compares the token's audience against.
|
||||
|
|
|
|||
|
|
@ -106,6 +106,7 @@ let
|
|||
|
||||
forgeMetricsUrl = "https://forge.t.local/metrics";
|
||||
autheliaMetricsUrl = "https://auth.t.local/metrics";
|
||||
vmMetricsUrl = "https://metrics.t.local/metrics";
|
||||
|
||||
# The policy authelia applies to `subject` requesting `https://<domain><path>`,
|
||||
# where `path` is what authelia matches `resources` against: the cleaned path
|
||||
|
|
@ -251,6 +252,42 @@ let
|
|||
&& authz.proxyPass == "https://${h.swarm.authelia.domain}/api/authz/auth-request"
|
||||
&& lib.hasInfix "internal;" authz.extraConfig;
|
||||
}
|
||||
{
|
||||
# The store's half of the same move, on a collector host that does not
|
||||
# run the store.
|
||||
name = "a collector on a host without the metrics store scrapes the store by name";
|
||||
ok =
|
||||
let
|
||||
s = otelSettings otelRemoteAuthelia;
|
||||
job = lib.findFirst (
|
||||
j: j.job_name == "victoriametrics"
|
||||
) null s.receivers.prometheus.config.scrape_configs;
|
||||
in
|
||||
!otelRemoteAuthelia.services.hyperhive.deploy.victoriametrics.enable
|
||||
&& job != null
|
||||
&& job.static_configs == [ { targets = [ "metrics.t.local" ]; } ]
|
||||
&& job.metrics_path == "/metrics"
|
||||
&& job.oauth2.endpoint_params.audience == vmMetricsUrl;
|
||||
}
|
||||
{
|
||||
# The metrics vhost is default-deny; `/metrics` reaches the store only
|
||||
# through its own exact location, gated like the push route. The `/`
|
||||
# clauses pin that the catch-all still refuses every other path.
|
||||
name = "the metrics store's vhost serves /metrics only behind the collector's auth subrequest";
|
||||
ok =
|
||||
let
|
||||
h = allLocal.services.hyperhive;
|
||||
locs = allLocal.services.nginx.virtualHosts.${h.swarm.victoriametrics.domain}.locations;
|
||||
m = locs."= /metrics" or null;
|
||||
in
|
||||
m != null
|
||||
&& m.proxyPass == "http://127.0.0.1:${toString h.swarm.victoriametrics.port}/metrics"
|
||||
&& lib.hasInfix "auth_request /__metrics_push_authz;" m.extraConfig
|
||||
&& !(lib.hasInfix "error_page" m.extraConfig)
|
||||
&& lib.hasInfix "internal;" locs."= /__metrics_push_authz".extraConfig
|
||||
&& locs."/".return == "403"
|
||||
&& locs."/".proxyPass == null;
|
||||
}
|
||||
{
|
||||
# Every published target admits the collector's client and refuses any
|
||||
# other authenticated subject, with or without a query string. Read
|
||||
|
|
@ -283,6 +320,7 @@ let
|
|||
lib.sort lib.lessThan (lib.attrValues h.swarm.otel.publishedScrapeTargets) == [
|
||||
autheliaMetricsUrl
|
||||
forgeMetricsUrl
|
||||
vmMetricsUrl
|
||||
]
|
||||
&& lib.length targets == 4
|
||||
&& lib.all (t: policy collector t == "one_factor") targets
|
||||
|
|
@ -367,6 +405,23 @@ let
|
|||
&& lib.length jobs == 1
|
||||
&& (lib.head jobs).static_configs == [ { targets = [ "auth.t.local" ]; } ];
|
||||
}
|
||||
{
|
||||
# Same for the metrics store: one job, by its published name.
|
||||
name = "a host running the metrics store and the collector scrapes the store once, by name";
|
||||
ok =
|
||||
let
|
||||
o = allLocal.services.hyperhive.swarm.otel;
|
||||
jobs =
|
||||
lib.filter (j: j.job_name == "victoriametrics")
|
||||
(otelSettings allLocal).receivers.prometheus.config.scrape_configs;
|
||||
in
|
||||
allLocal.services.hyperhive.deploy.victoriametrics.enable
|
||||
&& allLocal.services.hyperhive.deploy.swarm-otel.enable
|
||||
&& !(o.scrapeTargets ? victoriametrics)
|
||||
&& (o.publishedScrapeTargets.victoriametrics or null) == vmMetricsUrl
|
||||
&& lib.length jobs == 1
|
||||
&& (lib.head jobs).static_configs == [ { targets = [ "metrics.t.local" ]; } ];
|
||||
}
|
||||
{
|
||||
# The case above reads the options; this one reads the assertion they
|
||||
# feed, which is what refuses an operator's `nixos-rebuild`.
|
||||
|
|
|
|||
Loading…
Reference in a new issue