From 338a767a61f469ca1264a048c2acf4c9e92301ba Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 13 Oct 2026 10:44:50 +0200 Subject: [PATCH] swarm-otel: scrape VictoriaMetrics through its published name The store's own /metrics was scraped over loopback, declared under deploy.victoriametrics.enable, so a collector on another host never had a job for it. Declare it as a published target on every host instead, keyed on swarm.victoriametrics.domain, and serve it on the metrics vhost behind the existing authz subrequest. Authelia's per-target rules (one collector rule, then deny) now cover it too. The loopback entry goes, so a single-host swarm scrapes the store once. --- docs/swarm/services.md | 5 +- nix/host-modules/swarm-otel-service.nix | 2 + nix/host-modules/swarm-victoriametrics.nix | 39 +++++++-------- nix/module-eval/swarm-otel-identity.nix | 55 ++++++++++++++++++++++ 4 files changed, 80 insertions(+), 21 deletions(-) diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 26628c8b..afb03b07 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -172,8 +172,9 @@ Neither container is reachable except through the gateway: both bind loopback, and VictoriaMetrics authenticates nothing itself. The `metrics.` vhost forwards only the store's read API (the Prometheus query, series, label and metadata routes, plus the vmui web UI) -without a credential. It forwards the collector's OTLP push only with an -authelia token, and refuses every other path, the store's write and admin +without a credential. It forwards the collector's OTLP push, and the +collector's scrape of the store's own `/metrics`, only with an authelia +token, and refuses every other path, the store's write and admin API included, with 403. The bare domain redirects to vmui. ### Logs (VictoriaLogs) diff --git a/nix/host-modules/swarm-otel-service.nix b/nix/host-modules/swarm-otel-service.nix index c63fe78b..65483bdd 100644 --- a/nix/host-modules/swarm-otel-service.nix +++ b/nix/host-modules/swarm-otel-service.nix @@ -358,5 +358,7 @@ in forgejo = "https://${hyperhiveCfg.swarm.forge.domain}/metrics"; # The vhost location is in ./swarm-authelia.nix. authelia = "https://${hyperhiveCfg.swarm.authelia.domain}/metrics"; + # The vhost location is in ./swarm-victoriametrics.nix. + victoriametrics = "https://${vmCfg.domain}/metrics"; }; } diff --git a/nix/host-modules/swarm-victoriametrics.nix b/nix/host-modules/swarm-victoriametrics.nix index 63164a2b..a9da3e21 100644 --- a/nix/host-modules/swarm-victoriametrics.nix +++ b/nix/host-modules/swarm-victoriametrics.nix @@ -116,22 +116,6 @@ in services.hyperhive.gateway.enable = lib.mkDefault true; services.hyperhive.gateway.dns.enable = lib.mkDefault true; - # This store publishes its own health as prometheus metrics on the same - # listener it serves queries on, so the swarm's collector can scrape it - # with no exporter and no extra port. - # - # Declared here rather than in the collector's module because that is the - # rule the option carries: an entry exists only where the service that - # named it runs, which is what keeps scraper and target on one host by - # construction instead of by luck. - # - # The loopback literal is safe for this option and NOT for the write path, - # which is the distinction that matters now that the two differ. A scrape - # target is only ever read by a collector on this host, so loopback states - # a fact. The collector's push goes to the swarm name through the gateway, - # because the collector need not be here at all. - services.hyperhive.swarm.otel.scrapeTargets.victoriametrics = "127.0.0.1:${toString cfg.port}"; - services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // { listen = gatewayCfg.lib.listen; extraConfig = gatewayCfg.lib.securityHeaders; @@ -146,8 +130,8 @@ in # index page links mostly to paths refused above, so send it to vmui. "= /".return = "302 /vmui/"; - # The swarm collector's ingest route, and the only authenticated thing - # on this vhost. `=` so it outranks the `/` prefix above. + # The swarm collector's ingest route. `=` so it outranks the `/` + # prefix above. # # ⚠️ No `error_page 401 =302` here, and its absence is the point: a # redirect is right for a browser and wrong for a pusher, which would @@ -160,7 +144,24 @@ in ''; }; - # The subrequest. Same target and header set as the sibling log + # The store's own health, for the swarm collector wherever it runs: + # served on the query listener, so no exporter and no extra port. `=` + # for the same reason as the push route, and no `error_page` for the + # same reason: a scraper would parse a login page as metrics. + # + # The collector requests exactly this URL, declared as + # `publishedScrapeTargets.victoriametrics` in ./swarm-otel-service.nix; + # the two must agree, since that string is also the token's audience. + "= /metrics" = { + proxyPass = "http://127.0.0.1:${toString cfg.port}/metrics"; + extraConfig = '' + auth_request /__metrics_push_authz; + ''; + }; + + # The subrequest for both gated locations above. It carries the + # requested URL, and authelia decides per URL, so the scrape needs no + # subrequest of its own. Same target and header set as the sibling log # store's, which took them from `swarm-ui.nix` — `X-Original-URL` and # `X-Original-Method` are what authelia's auth-request implementation # reads, and the address it compares the token's audience against. diff --git a/nix/module-eval/swarm-otel-identity.nix b/nix/module-eval/swarm-otel-identity.nix index 5303a992..fc626023 100644 --- a/nix/module-eval/swarm-otel-identity.nix +++ b/nix/module-eval/swarm-otel-identity.nix @@ -106,6 +106,7 @@ let forgeMetricsUrl = "https://forge.t.local/metrics"; autheliaMetricsUrl = "https://auth.t.local/metrics"; + vmMetricsUrl = "https://metrics.t.local/metrics"; # The policy authelia applies to `subject` requesting `https://`, # where `path` is what authelia matches `resources` against: the cleaned path @@ -251,6 +252,42 @@ let && authz.proxyPass == "https://${h.swarm.authelia.domain}/api/authz/auth-request" && lib.hasInfix "internal;" authz.extraConfig; } + { + # The store's half of the same move, on a collector host that does not + # run the store. + name = "a collector on a host without the metrics store scrapes the store by name"; + ok = + let + s = otelSettings otelRemoteAuthelia; + job = lib.findFirst ( + j: j.job_name == "victoriametrics" + ) null s.receivers.prometheus.config.scrape_configs; + in + !otelRemoteAuthelia.services.hyperhive.deploy.victoriametrics.enable + && job != null + && job.static_configs == [ { targets = [ "metrics.t.local" ]; } ] + && job.metrics_path == "/metrics" + && job.oauth2.endpoint_params.audience == vmMetricsUrl; + } + { + # The metrics vhost is default-deny; `/metrics` reaches the store only + # through its own exact location, gated like the push route. The `/` + # clauses pin that the catch-all still refuses every other path. + name = "the metrics store's vhost serves /metrics only behind the collector's auth subrequest"; + ok = + let + h = allLocal.services.hyperhive; + locs = allLocal.services.nginx.virtualHosts.${h.swarm.victoriametrics.domain}.locations; + m = locs."= /metrics" or null; + in + m != null + && m.proxyPass == "http://127.0.0.1:${toString h.swarm.victoriametrics.port}/metrics" + && lib.hasInfix "auth_request /__metrics_push_authz;" m.extraConfig + && !(lib.hasInfix "error_page" m.extraConfig) + && lib.hasInfix "internal;" locs."= /__metrics_push_authz".extraConfig + && locs."/".return == "403" + && locs."/".proxyPass == null; + } { # Every published target admits the collector's client and refuses any # other authenticated subject, with or without a query string. Read @@ -283,6 +320,7 @@ let lib.sort lib.lessThan (lib.attrValues h.swarm.otel.publishedScrapeTargets) == [ autheliaMetricsUrl forgeMetricsUrl + vmMetricsUrl ] && lib.length targets == 4 && lib.all (t: policy collector t == "one_factor") targets @@ -367,6 +405,23 @@ let && lib.length jobs == 1 && (lib.head jobs).static_configs == [ { targets = [ "auth.t.local" ]; } ]; } + { + # Same for the metrics store: one job, by its published name. + name = "a host running the metrics store and the collector scrapes the store once, by name"; + ok = + let + o = allLocal.services.hyperhive.swarm.otel; + jobs = + lib.filter (j: j.job_name == "victoriametrics") + (otelSettings allLocal).receivers.prometheus.config.scrape_configs; + in + allLocal.services.hyperhive.deploy.victoriametrics.enable + && allLocal.services.hyperhive.deploy.swarm-otel.enable + && !(o.scrapeTargets ? victoriametrics) + && (o.publishedScrapeTargets.victoriametrics or null) == vmMetricsUrl + && lib.length jobs == 1 + && (lib.head jobs).static_configs == [ { targets = [ "metrics.t.local" ]; } ]; + } { # The case above reads the options; this one reads the assertion they # feed, which is what refuses an operator's `nixos-rebuild`.