vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-otel: scrape grafana through its published name

Browse source 
Grafana's own metrics were scraped over loopback through the
`grafana-metrics` nginx vhost, declared only where grafana and the
collector share a host, so a collector on any other host had no job for
it. Its target is now declared on every host as
`https://<swarm.grafana.domain>/metrics`, next to the other published
targets, and grafana's vhost serves `= /metrics` behind the collector's
auth subrequest (`/__grafana_metrics_authz`, the forge's shape). The
rules derived from `publishedScrapeTargets` admit the collector's client
there and deny every other subject.

The loopback scrape is gone with the vhost that served it and its
`swarm.grafana.metricsPort` option, so a single host still scrapes
grafana once, now by name. A config that sets that option now fails
evaluation with "option does not exist".

On the published vhost `/metrics` was answered with 403 for everyone.
It now answers the collector's token, and a plain session gets authelia's
`deny`. The design note that ruled the published vhost out for scraping
predates the collector's credential; it is replaced by the location's
own comment and a line on `swarm.grafana.domain`.

Part of #5205
This commit is contained in:
atlas 2026-10-14 19:13:53 +02:00 • committed by mara
commit 54f6061d41
7 changed files with 145 additions and 99 deletions
Download patch file Download diff file Expand all files Collapse all files

9
nix/module-eval/forge-placement.nix
View file

@ -138,16 +138,15 @@ let
&& !(refusedOver bare "deploy.forgejo.sso.clientSecretFile");
}
{
# Authelia is a published target; ./swarm-otel-identity.nix covers it.
name = "a host running the collector, authelia, grafana and nats scrapes grafana and nats on loopback, authelia not";
# Authelia and grafana are published targets; ./swarm-otel-identity.nix
# covers them.
name = "a host running the collector, authelia, grafana and nats scrapes nats on loopback, authelia and grafana not";
ok =
let
t = servicesHere.services.hyperhive.swarm.otel.scrapeTargets;
port = n: toString servicesHere.services.hyperhive.swarm.${n}.metricsPort;
in
t.grafana == "127.0.0.1:${port "grafana"}"
&& t.nats == "127.0.0.1:${port "nats"}"
&& !(t ? authelia);
t.nats == "127.0.0.1:${port "nats"}" && !(t ? authelia) && !(t ? grafana);
}
{
name = "hosting the swarm's shared services runs the forge";

26
nix/module-eval/grafana.nix
View file

@ -109,12 +109,6 @@ let
&& lib.hasInfix option a.message
) m.assertions;
cases = [
{
name = "grafana declares no scrape target on a host with no collector";
ok =
!(grafanaOldPath.services.hyperhive.deploy.swarm-otel.enable)
&& !(grafanaOldPath.services.hyperhive.swarm.otel.scrapeTargets ? grafana);
}
{
# The socket's directory is its access control, so it must stay dedicated
# to this one socket. `readOnly` is what makes that inexpressible: a
@ -124,22 +118,30 @@ let
ok =
!(builtins.tryEval grafanaSocketDirSet.services.hyperhive.deploy.grafana.socketDir).success
&&
grafanaOldPath.services.nginx.virtualHosts."grafana-metrics".locations."= /metrics".proxyPass
grafanaOldPath.services.nginx.virtualHosts.${grafanaOldPath.services.hyperhive.swarm.grafana.domain}.locations."= /metrics".proxyPass
== "http://unix:/run/swarm-grafana/grafana.sock:/metrics";
}
{
# Grafana serves `/metrics` unauthenticated, so the published name must
# answer it itself; the scrape reads the loopback vhost above instead.
# Grafana serves `/metrics` unauthenticated, so the published name gates
# it on the collector's auth subrequest: no `error_page 401 =302` (a
# scraper would parse a login page as metrics) and no `/__hive_authelia`.
# `=` is what catches the normalised variants and still leaves the rest
# of Grafana proxied.
name = "grafana's published vhost refuses /metrics and still proxies everything else";
name = "grafana's published vhost serves /metrics only behind the collector's auth subrequest";
ok =
let
g = grafanaOldPath.services.hyperhive.swarm.grafana;
locs = grafanaOldPath.services.nginx.virtualHosts.${g.domain}.locations;
checksOf = l: lib.filter (lib.hasPrefix "auth_request") (lib.splitString "\n" l.extraConfig);
authz = locs."= /__grafana_metrics_authz";
in
locs."= /metrics".return == "403"
&& locs."= /metrics".proxyPass == null
locs ? "= /metrics"
&& locs."= /metrics".return == null
&& checksOf locs."= /metrics" == [ "auth_request /__grafana_metrics_authz;" ]
&& !(lib.hasInfix "error_page" locs."= /metrics".extraConfig)
&& authz.proxyPass == "https://auth.t.local/api/authz/auth-request"
&& lib.hasInfix "internal;" authz.extraConfig
&& !(grafanaOldPath.services.nginx.virtualHosts ? "grafana-metrics")
&& lib.hasPrefix "http://unix:" locs."/".proxyPass;
}
{

63
nix/module-eval/swarm-otel-identity.nix
View file

@ -108,6 +108,7 @@ let
forgeMetricsUrl = "https://forge.t.local/metrics";
autheliaMetricsUrl = "https://auth.t.local/metrics";
grafanaMetricsUrl = "https://grafana.t.local/metrics";
logsMetricsUrl = "https://logs.t.local/metrics";
vmMetricsUrl = "https://metrics.t.local/metrics";
@ -374,10 +375,11 @@ let
lib.sort lib.lessThan (lib.attrValues h.swarm.otel.publishedScrapeTargets) == [
autheliaMetricsUrl
forgeMetricsUrl
grafanaMetricsUrl
logsMetricsUrl
vmMetricsUrl
]
&& lib.length targets == 8
&& lib.length targets == 10
&& lib.all (t: policy collector t == "one_factor") targets
&& lib.all (t: policy "user:someone" t == "deny") targets
&& lib.all (t: policy "group:admins" t == "deny") targets;
@ -425,7 +427,7 @@ let
];
matched = row: builtins.match re row.path != null;
in
lib.length metricsRules == 6
lib.length metricsRules == 8
&& lib.all (r: r.resources == [ re ]) metricsRules
&& re == "^/metrics([?].*)?$"
&& lib.all (row: matched row == (row.nginx == "= /metrics")) table;
@ -562,6 +564,63 @@ let
&& lib.length jobs == 1
&& (lib.head jobs).static_configs == [ { targets = [ "logs.t.local" ]; } ];
}
{
name = "a collector on a host without grafana scrapes it by name";
ok =
let
s = otelSettings otelRemoteAuthelia;
job = lib.findFirst (j: j.job_name == "grafana") null s.receivers.prometheus.config.scrape_configs;
in
!otelRemoteAuthelia.services.hyperhive.deploy.grafana.enable
&& job != null
&& job.static_configs == [ { targets = [ "grafana.t.local" ]; } ]
&& job.metrics_path == "/metrics"
&& job.oauth2.endpoint_params.audience == grafanaMetricsUrl;
}
{
# The `deny` that confines grafana's `/metrics` is scoped to that path,
# and comes after the collector's rule: every other path on the name
# still reaches `default_policy`, as it did with no rule at all.
name = "authelia confines grafana's /metrics to the collector and leaves its other paths alone";
ok =
let
h = autheliaOnly.services.hyperhive;
collector = "oauth2:client:${h.swarm.otel.clientId}";
rules =
autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules;
grafanaRules = lib.filter (r: r.domain == "grafana.t.local") rules;
policy = subject: autheliaPolicy autheliaOnly subject "grafana.t.local";
in
map (r: r.policy) grafanaRules == [
"one_factor"
"deny"
]
&& lib.all (r: r.resources == [ "^/metrics([?].*)?$" ]) grafanaRules
&& (lib.head grafanaRules).subject == [ collector ]
&& policy collector "/metrics" == "one_factor"
&& policy "user:someone" "/metrics" == "deny"
&& policy "user:someone" "/metrics?x=1" == "deny"
&& policy "group:admins" "/metrics" == "deny"
&& policy "user:someone" "/" == "one_factor"
&& policy "user:someone" "/login/generic_oauth" == "one_factor";
}
{
# Single host: grafana is scraped once, by its published name.
name = "a host running grafana and the collector scrapes it once, by name";
ok =
let
o = allLocal.services.hyperhive.swarm.otel;
jobs =
lib.filter (j: j.job_name == "grafana")
(otelSettings allLocal).receivers.prometheus.config.scrape_configs;
in
allLocal.services.hyperhive.deploy.grafana.enable
&& allLocal.services.hyperhive.deploy.swarm-otel.enable
&& !(o.scrapeTargets ? grafana)
&& (o.publishedScrapeTargets.grafana or null) == grafanaMetricsUrl
&& lib.length jobs == 1
&& (lib.head jobs).static_configs == [ { targets = [ "grafana.t.local" ]; } ];
}
{
# Read against the roster the fixture declares rather than against
# names spelled here: an arm naming `h1` passes on a single-hive