From f5e99fe2487f3dcbe2806a673e142ab4d758dd69 Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 14 Nov 2026 22:05:08 +0100 Subject: [PATCH] gateway: cache authelia's 200 verdicts on the log and metric ingest routes --- docs/4-implementation/swarm/telemetry.md | 4 ++ nix/host-modules/swarm-victorialogs.nix | 58 +++++++++++++++++++- nix/host-modules/swarm-victoriametrics.nix | 61 ++++++++++++++++++++-- nix/module-eval/core-toggle.nix | 45 ++++++++++++++++ nix/module-eval/swarm-otel-identity.nix | 4 +- 5 files changed, 167 insertions(+), 5 deletions(-) diff --git a/docs/4-implementation/swarm/telemetry.md b/docs/4-implementation/swarm/telemetry.md index 02ac26562..786478fa4 100644 --- a/docs/4-implementation/swarm/telemetry.md +++ b/docs/4-implementation/swarm/telemetry.md @@ -124,6 +124,8 @@ vhost does it: any other session or client there, operators included. - **Every other path**, the store's write and admin API included: 403. +The gateway remembers authelia's answer for the collector's push route. It keys the answer on the request's `Authorization` header and the route, keeps only a 200, and keeps it for 60 seconds. A burst of pushes from one token costs authelia one subrequest, and a push that finds its verdict in the cache doesn't wait on authelia, however slow it is. A 401 or 403 always goes to authelia again. A request without an `Authorization` header skips the cache, and the subrequest drops the `Cookie` header. A revoked or expired token keeps pushing for up to 60 seconds. The `/metrics` scrape and the other routes ask authelia every time. + `swarm-metrics-auth` and `swarm-metrics-agent-auth` read the store with one leaf, `deploy.victoriametrics.authBaoClientCertFile` / `authBaoClientKeyFile`. Where the secret store runs on the same host, these default to the leaf it @@ -210,6 +212,8 @@ collector's client to `/metrics` and the ingest route `/insert/opentelemetry/v1/logs`; any other session gets `deny` on both. The read side stays open to any authenticated caller. +The log store's ingest route `/insert/opentelemetry/v1/logs` gets the same 60-second cache of authelia's 200 answers as the metrics store's push route, with the same revocation window. The log store's other routes ask authelia every time. + ## Telemetry collector (OTEL) The swarm's collector receives from every hive's own collector and is the diff --git a/nix/host-modules/swarm-victorialogs.nix b/nix/host-modules/swarm-victorialogs.nix index 8666fa24a..fbcfc7f01 100644 --- a/nix/host-modules/swarm-victorialogs.nix +++ b/nix/host-modules/swarm-victorialogs.nix @@ -52,6 +52,29 @@ let error_page 401 =302 https://${hyperhiveCfg.swarm.authelia.domain}/?rd=$target_url; ''; + # Cache directives for the ingest route's authelia subrequest. The key is + # the `Authorization` header and the route, and only a 200 is stored, so a + # 401 or 403 always reaches authelia again. A request without an + # `Authorization` header skips the cache both ways + # (`swarm_logs_ingest_nocache` in `appendHttpConfig`), so a cached grant is + # never read by a caller that presented no token. The `Cookie` header is + # dropped from the subrequest, because authelia also accepts a session + # cookie and the key does not cover it. `proxy_ignore_headers` makes the + # `proxy_cache_valid` line the only lifetime, whatever cache headers + # authelia sends. + ingestAuthzCache = '' + proxy_set_header Cookie ""; + proxy_cache swarm_logs_ingest_authz; + proxy_cache_key "$http_authorization|$http_host$request_uri"; + proxy_cache_valid 200 60s; + proxy_cache_lock on; + proxy_cache_lock_timeout 30s; + proxy_cache_use_stale updating; + proxy_cache_bypass $swarm_logs_ingest_nocache; + proxy_no_cache $swarm_logs_ingest_nocache; + proxy_ignore_headers Cache-Control Expires Set-Cookie Vary; + ''; + # The agents' `auth_request` responder (`swarm-logs-auth`). Its socket's # directory holds that socket alone and is traverse-only for others, the # shape `swarm-controller.nix` gives its own. @@ -326,8 +349,21 @@ in default /__hive_authelia; } limit_req_zone $binary_remote_addr zone=swarm_logs_agent_auth:1m rate=${deployCfg.victorialogs.authRate}; + map $http_authorization $swarm_logs_ingest_nocache { + "" 1; + default 0; + } ''; + # The zone behind `ingestAuthzCache`. + services.nginx.proxyCachePath.swarm_logs_ingest_authz = { + enable = true; + keysZoneName = "swarm_logs_ingest_authz"; + keysZoneSize = "1m"; + maxSize = "10m"; + inactive = "10m"; + }; + # The gateway name, inside `deployCfg.victorialogs.enable` — same # "only the host that runs the service may claim the name" guard every # sibling swarm-service module uses (`swarm-grafana.nix`, @@ -372,7 +408,7 @@ in "= /insert/opentelemetry/v1/logs" = { proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/insert/opentelemetry/v1/logs"; extraConfig = '' - auth_request /__hive_authelia; + auth_request /__swarm_logs_ingest_authz; ''; }; @@ -466,6 +502,26 @@ in ''; }; + # The ingest route's copy of the subrequest below, plus the verdict + # cache. Separate because `/__hive_authelia` also serves the browser + # and scrape routes, which must not be cached. + "= /__swarm_logs_ingest_authz" = { + proxyPass = "https://${hyperhiveCfg.swarm.authelia.domain}/api/authz/auth-request"; + recommendedProxySettings = false; + extraConfig = '' + internal; + ${gatewayCfg.lib.verifiedProxyTo hyperhiveCfg.swarm.authelia.domain} + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-Method $request_method; + proxy_set_header X-Original-URL $scheme://$http_host$request_uri; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + ${ingestAuthzCache} + ''; + }; + # The subrequest itself — same target, same header set, same # reasoning as `swarm-ui.nix`'s own copy (measured against the # pinned authelia binary, not copied from an example). diff --git a/nix/host-modules/swarm-victoriametrics.nix b/nix/host-modules/swarm-victoriametrics.nix index 2b81aaf2b..c44cf976a 100644 --- a/nix/host-modules/swarm-victoriametrics.nix +++ b/nix/host-modules/swarm-victoriametrics.nix @@ -70,6 +70,28 @@ let # The store serves only `/values` under `/api/v1/label/`. readPrefixes = [ "/api/v1/label/" ]; + # Cache directives for the push route's authelia subrequest. The key is the + # `Authorization` header and the route, and only a 200 is stored, so a 401 + # or 403 always reaches authelia again. A request without an `Authorization` + # header skips the cache both ways (`swarm_metrics_ingest_nocache` in + # `appendHttpConfig`), so a cached grant is never read by a caller that + # presented no token. The `Cookie` header is dropped from the subrequest, + # because authelia also accepts a session cookie and the key does not cover + # it. `proxy_ignore_headers` makes the `proxy_cache_valid` line the only + # lifetime, whatever cache headers authelia sends. + ingestAuthzCache = '' + proxy_set_header Cookie ""; + proxy_cache swarm_metrics_ingest_authz; + proxy_cache_key "$http_authorization|$http_host$request_uri"; + proxy_cache_valid 200 60s; + proxy_cache_lock on; + proxy_cache_lock_timeout 30s; + proxy_cache_use_stale updating; + proxy_cache_bypass $swarm_metrics_ingest_nocache; + proxy_no_cache $swarm_metrics_ingest_nocache; + proxy_ignore_headers Cache-Control Expires Set-Cookie Vary; + ''; + # Bare `auth_request`, no login redirect: a machine caller handed # `error_page 401 =302` follows it and reads authelia's login page as a 200 # holding no series. `/__metrics_read_auth` picks the check from the @@ -397,8 +419,21 @@ in default /__metrics_authelia; } limit_req_zone $binary_remote_addr zone=swarm_metrics_agent_auth:1m rate=${deployCfg.victoriametrics.agentAuthRate}; + map $http_authorization $swarm_metrics_ingest_nocache { + "" 1; + default 0; + } ''; + # The zone behind `ingestAuthzCache`. + services.nginx.proxyCachePath.swarm_metrics_ingest_authz = { + enable = true; + keysZoneName = "swarm_metrics_ingest_authz"; + keysZoneSize = "1m"; + maxSize = "10m"; + inactive = "10m"; + }; + # `forceSSL`: authelia answers a plain-http auth subrequest # with 400, which `auth_request` cannot read as anything but a broken # check. Same as ./swarm-victorialogs.nix's vhost. @@ -428,7 +463,7 @@ in "= /opentelemetry/api/v1/push" = { proxyPass = upstream "/opentelemetry/api/v1/push"; extraConfig = '' - auth_request /__metrics_authelia; + auth_request /__metrics_ingest_authz; ''; }; @@ -485,8 +520,28 @@ in ''; }; - # The authelia subrequest. For the collector's two locations above it - # admits only the collector's client; for the read and vmui locations + # The push route's copy of the subrequest below, plus the verdict + # cache. Separate because `/__metrics_authelia` also serves `/metrics` + # and vmui, which must not be cached. + "= /__metrics_ingest_authz" = { + proxyPass = "https://${hyperhiveCfg.swarm.authelia.domain}/api/authz/auth-request"; + recommendedProxySettings = false; + extraConfig = '' + internal; + ${gatewayCfg.lib.verifiedProxyTo hyperhiveCfg.swarm.authelia.domain} + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-Method $request_method; + proxy_set_header X-Original-URL $scheme://$http_host$request_uri; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + ${ingestAuthzCache} + ''; + }; + + # The authelia subrequest. For `/metrics` it admits only the collector's + # client; for the read and vmui locations # it admits an operator's session (`metricsStoreRules` in # ./swarm-authelia.nix). Same target and header set as the sibling log # store's, which took them from `swarm-ui.nix` — `X-Original-URL` and diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index f4e8aa60e..a5ff7c730 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -1113,6 +1113,51 @@ let && !(lib.hasInfix "error_page" l) && lib.hasInfix "error_page" browser; } + { + # The two ingest routes ask authelia through a subrequest that caches a + # 200 for a minute, keyed on the bearer and the route. Everything else + # that asks authelia (the browser routes, `/metrics`) keeps the plain + # subrequest: the `!` arms on `/__hive_authelia` and `/__metrics_authelia` + # are paired with the positive arms on the ingest subrequests, so a + # misspelt location name cannot pass. + name = "the ingest routes cache authelia's 200 verdicts per bearer and no other authelia subrequest does"; + ok = + let + nginx = allLocal.services.nginx; + http = nginx.appendHttpConfig; + logs = nginx.virtualHosts."logs.t.local".locations; + metrics = nginx.virtualHosts."metrics.t.local".locations; + cached = + zone: nocache: loc: + lib.hasInfix "proxy_cache ${zone};" loc.extraConfig + && lib.hasInfix ''proxy_cache_key "$http_authorization|$http_host$request_uri";'' loc.extraConfig + && lib.hasInfix "proxy_cache_valid 200 60s;" loc.extraConfig + && !(lib.hasInfix "proxy_cache_valid 401" loc.extraConfig) + && !(lib.hasInfix "proxy_cache_valid any" loc.extraConfig) + && lib.hasInfix "proxy_cache_lock on;" loc.extraConfig + && lib.hasInfix "proxy_cache_bypass ${nocache};" loc.extraConfig + && lib.hasInfix "proxy_no_cache ${nocache};" loc.extraConfig + && lib.hasInfix ''proxy_set_header Cookie "";'' loc.extraConfig + && lib.hasInfix "internal;" loc.extraConfig; + in + lib.hasInfix "auth_request /__swarm_logs_ingest_authz;" + logs."= /insert/opentelemetry/v1/logs".extraConfig + && + lib.hasInfix "auth_request /__metrics_ingest_authz;" + metrics."= /opentelemetry/api/v1/push".extraConfig + && cached "swarm_logs_ingest_authz" "$swarm_logs_ingest_nocache" logs."= /__swarm_logs_ingest_authz" + && + cached "swarm_metrics_ingest_authz" "$swarm_metrics_ingest_nocache" + metrics."= /__metrics_ingest_authz" + && lib.hasInfix "map $http_authorization $swarm_logs_ingest_nocache" http + && lib.hasInfix "map $http_authorization $swarm_metrics_ingest_nocache" http + && nginx.proxyCachePath.swarm_logs_ingest_authz.enable + && nginx.proxyCachePath.swarm_metrics_ingest_authz.enable + && !(lib.hasInfix "proxy_cache" logs."= /__hive_authelia".extraConfig) + && !(lib.hasInfix "proxy_cache" logs."/".extraConfig) + && !(lib.hasInfix "proxy_cache" metrics."= /__metrics_authelia".extraConfig) + && !(lib.hasInfix "proxy_cache" metrics."= /metrics".extraConfig); + } { # The store authenticates nothing itself, so a location that forwards to # it without `auth_request` is open to anything that can reach the diff --git a/nix/module-eval/swarm-otel-identity.nix b/nix/module-eval/swarm-otel-identity.nix index 83588c373..d1b110e70 100644 --- a/nix/module-eval/swarm-otel-identity.nix +++ b/nix/module-eval/swarm-otel-identity.nix @@ -329,7 +329,9 @@ let && lib.all (p: policy "user:someone" p == "deny") (collectorPaths ++ readPaths) && lib.all (p: policy "group:admins" p == "deny") collectorPaths && lib.all (p: policy "group:admins" p == "one_factor") readPaths - && lib.hasInfix "auth_request /__metrics_authelia;" locs."= /opentelemetry/api/v1/push".extraConfig + && + lib.hasInfix "auth_request /__metrics_ingest_authz;" + locs."= /opentelemetry/api/v1/push".extraConfig && lib.hasInfix "auth_request /__metrics_authelia;" locs."= /metrics".extraConfig; } {