mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

gateway: cache authelia's 200 verdicts on the log and metric ingest routes

Browse source
This commit is contained in:
atlas 2026-11-14 22:05:08 +01:00
commit f5e99fe248
5 changed files with 167 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

4
docs/4-implementation/swarm/telemetry.md
View file

@ -124,6 +124,8 @@ vhost does it:
any other session or client there, operators included.
- **Every other path**, the store's write and admin API included: 403.
The gateway remembers authelia's answer for the collector's push route. It keys the answer on the request's `Authorization` header and the route, keeps only a 200, and keeps it for 60 seconds. A burst of pushes from one token costs authelia one subrequest, and a push that finds its verdict in the cache doesn't wait on authelia, however slow it is. A 401 or 403 always goes to authelia again. A request without an `Authorization` header skips the cache, and the subrequest drops the `Cookie` header. A revoked or expired token keeps pushing for up to 60 seconds. The `/metrics` scrape and the other routes ask authelia every time.
`swarm-metrics-auth` and `swarm-metrics-agent-auth` read the store with one
leaf, `deploy.victoriametrics.authBaoClientCertFile` / `authBaoClientKeyFile`.
Where the secret store runs on the same host, these default to the leaf it
@ -210,6 +212,8 @@ collector's client to `/metrics` and the ingest route
`/insert/opentelemetry/v1/logs`; any other session gets `deny` on both. The
read side stays open to any authenticated caller.
The log store's ingest route `/insert/opentelemetry/v1/logs` gets the same 60-second cache of authelia's 200 answers as the metrics store's push route, with the same revocation window. The log store's other routes ask authelia every time.
## Telemetry collector (OTEL)
The swarm's collector receives from every hive's own collector and is the

58
nix/host-modules/swarm-victorialogs.nix
View file

@ -52,6 +52,29 @@ let
error_page 401 =302 https://${hyperhiveCfg.swarm.authelia.domain}/?rd=$target_url;
'';
# Cache directives for the ingest route's authelia subrequest. The key is
# the `Authorization` header and the route, and only a 200 is stored, so a
# 401 or 403 always reaches authelia again. A request without an
# `Authorization` header skips the cache both ways
# (`swarm_logs_ingest_nocache` in `appendHttpConfig`), so a cached grant is
# never read by a caller that presented no token. The `Cookie` header is
# dropped from the subrequest, because authelia also accepts a session
# cookie and the key does not cover it. `proxy_ignore_headers` makes the
# `proxy_cache_valid` line the only lifetime, whatever cache headers
# authelia sends.
ingestAuthzCache = ''
proxy_set_header Cookie "";
proxy_cache swarm_logs_ingest_authz;
proxy_cache_key "$http_authorization|$http_host$request_uri";
proxy_cache_valid 200 60s;
proxy_cache_lock on;
proxy_cache_lock_timeout 30s;
proxy_cache_use_stale updating;
proxy_cache_bypass $swarm_logs_ingest_nocache;
proxy_no_cache $swarm_logs_ingest_nocache;
proxy_ignore_headers Cache-Control Expires Set-Cookie Vary;
'';
# The agents' `auth_request` responder (`swarm-logs-auth`). Its socket's
# directory holds that socket alone and is traverse-only for others, the
# shape `swarm-controller.nix` gives its own.
@ -326,8 +349,21 @@ in
default /__hive_authelia;
}
limit_req_zone $binary_remote_addr zone=swarm_logs_agent_auth:1m rate=${deployCfg.victorialogs.authRate};
map $http_authorization $swarm_logs_ingest_nocache {
"" 1;
default 0;
}
'';
# The zone behind `ingestAuthzCache`.
services.nginx.proxyCachePath.swarm_logs_ingest_authz = {
enable = true;
keysZoneName = "swarm_logs_ingest_authz";
keysZoneSize = "1m";
maxSize = "10m";
inactive = "10m";
};
# The gateway name, inside `deployCfg.victorialogs.enable` — same
# "only the host that runs the service may claim the name" guard every
# sibling swarm-service module uses (`swarm-grafana.nix`,
@ -372,7 +408,7 @@ in
"= /insert/opentelemetry/v1/logs" = {
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/insert/opentelemetry/v1/logs";
extraConfig = ''
auth_request /__hive_authelia;
auth_request /__swarm_logs_ingest_authz;
'';
};
@ -466,6 +502,26 @@ in
'';
};
# The ingest route's copy of the subrequest below, plus the verdict
# cache. Separate because `/__hive_authelia` also serves the browser
# and scrape routes, which must not be cached.
"= /__swarm_logs_ingest_authz" = {
proxyPass = "https://${hyperhiveCfg.swarm.authelia.domain}/api/authz/auth-request";
recommendedProxySettings = false;
extraConfig = ''
internal;
${gatewayCfg.lib.verifiedProxyTo hyperhiveCfg.swarm.authelia.domain}
proxy_pass_request_body off;
proxy_set_header Content-Length "";
proxy_set_header X-Original-Method $request_method;
proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
${ingestAuthzCache}
'';
};
# The subrequest itself — same target, same header set, same
# reasoning as `swarm-ui.nix`'s own copy (measured against the
# pinned authelia binary, not copied from an example).

61
nix/host-modules/swarm-victoriametrics.nix
View file

@ -70,6 +70,28 @@ let
# The store serves only `<name>/values` under `/api/v1/label/`.
readPrefixes = [ "/api/v1/label/" ];
# Cache directives for the push route's authelia subrequest. The key is the
# `Authorization` header and the route, and only a 200 is stored, so a 401
# or 403 always reaches authelia again. A request without an `Authorization`
# header skips the cache both ways (`swarm_metrics_ingest_nocache` in
# `appendHttpConfig`), so a cached grant is never read by a caller that
# presented no token. The `Cookie` header is dropped from the subrequest,
# because authelia also accepts a session cookie and the key does not cover
# it. `proxy_ignore_headers` makes the `proxy_cache_valid` line the only
# lifetime, whatever cache headers authelia sends.
ingestAuthzCache = ''
proxy_set_header Cookie "";
proxy_cache swarm_metrics_ingest_authz;
proxy_cache_key "$http_authorization|$http_host$request_uri";
proxy_cache_valid 200 60s;
proxy_cache_lock on;
proxy_cache_lock_timeout 30s;
proxy_cache_use_stale updating;
proxy_cache_bypass $swarm_metrics_ingest_nocache;
proxy_no_cache $swarm_metrics_ingest_nocache;
proxy_ignore_headers Cache-Control Expires Set-Cookie Vary;
'';
# Bare `auth_request`, no login redirect: a machine caller handed
# `error_page 401 =302` follows it and reads authelia's login page as a 200
# holding no series. `/__metrics_read_auth` picks the check from the
@ -397,8 +419,21 @@ in
default /__metrics_authelia;
}
limit_req_zone $binary_remote_addr zone=swarm_metrics_agent_auth:1m rate=${deployCfg.victoriametrics.agentAuthRate};
map $http_authorization $swarm_metrics_ingest_nocache {
"" 1;
default 0;
}
'';
# The zone behind `ingestAuthzCache`.
services.nginx.proxyCachePath.swarm_metrics_ingest_authz = {
enable = true;
keysZoneName = "swarm_metrics_ingest_authz";
keysZoneSize = "1m";
maxSize = "10m";
inactive = "10m";
};
# `forceSSL`: authelia answers a plain-http auth subrequest
# with 400, which `auth_request` cannot read as anything but a broken
# check. Same as ./swarm-victorialogs.nix's vhost.
@ -428,7 +463,7 @@ in
"= /opentelemetry/api/v1/push" = {
proxyPass = upstream "/opentelemetry/api/v1/push";
extraConfig = ''
auth_request /__metrics_authelia;
auth_request /__metrics_ingest_authz;
'';
};
@ -485,8 +520,28 @@ in
'';
};
# The authelia subrequest. For the collector's two locations above it
# admits only the collector's client; for the read and vmui locations
# The push route's copy of the subrequest below, plus the verdict
# cache. Separate because `/__metrics_authelia` also serves `/metrics`
# and vmui, which must not be cached.
"= /__metrics_ingest_authz" = {
proxyPass = "https://${hyperhiveCfg.swarm.authelia.domain}/api/authz/auth-request";
recommendedProxySettings = false;
extraConfig = ''
internal;
${gatewayCfg.lib.verifiedProxyTo hyperhiveCfg.swarm.authelia.domain}
proxy_pass_request_body off;
proxy_set_header Content-Length "";
proxy_set_header X-Original-Method $request_method;
proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
${ingestAuthzCache}
'';
};
# The authelia subrequest. For `/metrics` it admits only the collector's
# client; for the read and vmui locations
# it admits an operator's session (`metricsStoreRules` in
# ./swarm-authelia.nix). Same target and header set as the sibling log
# store's, which took them from `swarm-ui.nix` — `X-Original-URL` and

45
nix/module-eval/core-toggle.nix
View file

@ -1113,6 +1113,51 @@ let
&& !(lib.hasInfix "error_page" l)
&& lib.hasInfix "error_page" browser;
}
{
# The two ingest routes ask authelia through a subrequest that caches a
# 200 for a minute, keyed on the bearer and the route. Everything else
# that asks authelia (the browser routes, `/metrics`) keeps the plain
# subrequest: the `!` arms on `/__hive_authelia` and `/__metrics_authelia`
# are paired with the positive arms on the ingest subrequests, so a
# misspelt location name cannot pass.
name = "the ingest routes cache authelia's 200 verdicts per bearer and no other authelia subrequest does";
ok =
let
nginx = allLocal.services.nginx;
http = nginx.appendHttpConfig;
logs = nginx.virtualHosts."logs.t.local".locations;
metrics = nginx.virtualHosts."metrics.t.local".locations;
cached =
zone: nocache: loc:
lib.hasInfix "proxy_cache ${zone};" loc.extraConfig
&& lib.hasInfix ''proxy_cache_key "$http_authorization|$http_host$request_uri";'' loc.extraConfig
&& lib.hasInfix "proxy_cache_valid 200 60s;" loc.extraConfig
&& !(lib.hasInfix "proxy_cache_valid 401" loc.extraConfig)
&& !(lib.hasInfix "proxy_cache_valid any" loc.extraConfig)
&& lib.hasInfix "proxy_cache_lock on;" loc.extraConfig
&& lib.hasInfix "proxy_cache_bypass ${nocache};" loc.extraConfig
&& lib.hasInfix "proxy_no_cache ${nocache};" loc.extraConfig
&& lib.hasInfix ''proxy_set_header Cookie "";'' loc.extraConfig
&& lib.hasInfix "internal;" loc.extraConfig;
in
lib.hasInfix "auth_request /__swarm_logs_ingest_authz;"
logs."= /insert/opentelemetry/v1/logs".extraConfig
&&
lib.hasInfix "auth_request /__metrics_ingest_authz;"
metrics."= /opentelemetry/api/v1/push".extraConfig
&& cached "swarm_logs_ingest_authz" "$swarm_logs_ingest_nocache" logs."= /__swarm_logs_ingest_authz"
&&
cached "swarm_metrics_ingest_authz" "$swarm_metrics_ingest_nocache"
metrics."= /__metrics_ingest_authz"
&& lib.hasInfix "map $http_authorization $swarm_logs_ingest_nocache" http
&& lib.hasInfix "map $http_authorization $swarm_metrics_ingest_nocache" http
&& nginx.proxyCachePath.swarm_logs_ingest_authz.enable
&& nginx.proxyCachePath.swarm_metrics_ingest_authz.enable
&& !(lib.hasInfix "proxy_cache" logs."= /__hive_authelia".extraConfig)
&& !(lib.hasInfix "proxy_cache" logs."/".extraConfig)
&& !(lib.hasInfix "proxy_cache" metrics."= /__metrics_authelia".extraConfig)
&& !(lib.hasInfix "proxy_cache" metrics."= /metrics".extraConfig);
}
{
# The store authenticates nothing itself, so a location that forwards to
# it without `auth_request` is open to anything that can reach the

4
nix/module-eval/swarm-otel-identity.nix
View file

@ -329,7 +329,9 @@ let
&& lib.all (p: policy "user:someone" p == "deny") (collectorPaths ++ readPaths)
&& lib.all (p: policy "group:admins" p == "deny") collectorPaths
&& lib.all (p: policy "group:admins" p == "one_factor") readPaths
&& lib.hasInfix "auth_request /__metrics_authelia;" locs."= /opentelemetry/api/v1/push".extraConfig
&&
lib.hasInfix "auth_request /__metrics_ingest_authz;"
locs."= /opentelemetry/api/v1/push".extraConfig
&& lib.hasInfix "auth_request /__metrics_authelia;" locs."= /metrics".extraConfig;
}
{