vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-logs-auth: rate-limit verification and cache positive verdicts

Browse source 
Any request on logs.<swarm> carrying a well-formed swarm-agent bearer
made swarm-logs-auth do a bao cert login and KV read before it could
refuse, with no cap on the rate.

nginx now applies limit_req (zone swarm_logs_agent_auth, keyed by
client address, 5r/s burst 20 nodelay by default) on the internal
location that forwards to swarm-logs-auth, answering 403 when over,
since auth_request passes through only 401/403 and swarm-logs retries
a 401. The authelia path and the query location carry no limit.

swarm-logs-auth remembers bearers it has granted, keyed by SHA-256 of
the bearer, for --verdict-ttl seconds (deploy.victorialogs.authVerdictTtl,
60 by default), holding at most 4096 and evicting the oldest. Denials
are never remembered. A replaced or deleted queue secret therefore
keeps working at the log store for up to the TTL after its last check.

Closes #5190
This commit is contained in:
atlas 2026-10-13 09:06:20 +02:00 • committed by mara
commit 8be4030d03
9 changed files with 367 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

50
nix/host-modules/swarm-victorialogs.nix
View file

@ -142,6 +142,46 @@ in
'';
};
options.services.hyperhive.deploy.victorialogs.authVerdictTtl = lib.mkOption {
type = lib.types.ints.unsigned;
default = 60;
description = ''
Seconds `swarm-logs-auth` grants an agent bearer it has verified again
without reading the secret store. A denial is never remembered.
This is also how long a re-minted or deleted queue secret keeps working
at the log store's query route after its last successful check. `0`
checks the store on every query.
'';
};
options.services.hyperhive.deploy.victorialogs.authRate = lib.mkOption {
type = lib.types.str;
default = "5r/s";
example = "30r/m";
description = ''
nginx `limit_req` rate, per client address, of `/select/logsql/`
requests carrying a `swarm-agent.` bearer, which nginx checks with
`swarm-logs-auth`. Each such request that the responder has not granted
within {option}`services.hyperhive.deploy.victorialogs.authVerdictTtl`
costs a secret store login and read, whether the bearer is right or not.
Counts every agent query, granted or not, so it is sized for the agents
that share one address querying at once, not for one agent. Requests
over the rate and its burst are answered 403.
'';
};
options.services.hyperhive.deploy.victorialogs.authBurst = lib.mkOption {
type = lib.types.ints.positive;
default = 20;
description = ''
Requests a client address may make above
{option}`services.hyperhive.deploy.victorialogs.authRate` before nginx
answers 403, served without delay.
'';
};
config = lib.mkIf deployCfg.victorialogs.enable {
assertions = [
{
@ -170,6 +210,7 @@ in
"~^Bearer swarm-agent\." /__swarm_logs_agent_auth;
default /__hive_authelia;
}
limit_req_zone $binary_remote_addr zone=swarm_logs_agent_auth:1m rate=${deployCfg.victorialogs.authRate};
'';
# This store publishes its own health as prometheus metrics on the same
@ -280,10 +321,18 @@ in
};
# The `Authorization` header rides the subrequest unchanged.
#
# Any well-formed agent bearer the responder has not recently granted
# costs a secret store login and read, so the rate per client address
# is capped here, ahead of the responder. 403 when over: the only
# statuses `auth_request` passes through are 401 and 403, and
# `swarm-logs` retries a 401.
"= /__swarm_logs_agent_auth" = {
proxyPass = "http://unix:${logsAuthSocket}:/";
extraConfig = ''
internal;
limit_req zone=swarm_logs_agent_auth burst=${toString deployCfg.victorialogs.authBurst} nodelay;
limit_req_status 403;
proxy_pass_request_body off;
proxy_set_header Content-Length "";
'';
@ -331,6 +380,7 @@ in
"--socket ${logsAuthSocket}"
# The role ./swarm-bao.nix writes on the store's `cert` mount.
"--store-cert-role swarm-logs-auth"
"--verdict-ttl ${toString deployCfg.victorialogs.authVerdictTtl}"
];
# Paths only: `argv` is world-readable via /proc/<pid>/cmdline.
LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") logsAuthStoreFiles;

21
nix/module-eval/core-toggle.nix
View file

@ -544,6 +544,27 @@ let
&& lib.hasInfix "--socket /run/swarm-logs-auth/auth.sock" unit.serviceConfig.ExecStart
&& l ? "= /__hive_authelia";
}
{
# Every agent bearer the responder has not recently granted costs a
# store login and read, so its location carries the per-address limit.
# The responder gets the grant TTL from the option. The query and
# authelia locations carry no limit of their own.
name = "agent bearer checks are rate-limited per address and granted for a TTL";
ok =
let
l = allLocal.services.nginx.virtualHosts."logs.t.local".locations;
http = allLocal.services.nginx.appendHttpConfig;
unit = allLocal.systemd.services.swarm-logs-auth;
in
lib.hasInfix "limit_req_zone $binary_remote_addr zone=swarm_logs_agent_auth:1m rate=5r/s;" http
&&
lib.hasInfix "limit_req zone=swarm_logs_agent_auth burst=20 nodelay;"
l."= /__swarm_logs_agent_auth".extraConfig
&& lib.hasInfix "limit_req_status 403;" l."= /__swarm_logs_agent_auth".extraConfig
&& !(lib.hasInfix "limit_req" l."^~ /select/logsql/".extraConfig)
&& !(lib.hasInfix "limit_req" l."= /__hive_authelia".extraConfig)
&& lib.hasInfix "--verdict-ttl 60" unit.serviceConfig.ExecStart;
}
{
# The responder reads the store as itself, with the leaf the store's
# host mints for it.