From bff605035cc7f7c8d3c1951423eeb16f51d55508 Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 13 Oct 2026 10:09:01 +0200 Subject: [PATCH] swarm-victorialogs: size the agent-auth limit for a host booting its agents together All agents on a host reach the gateway from one address and share one limit_req budget; a host boot is one cache miss per agent at once. Defaults go from 5r/s burst 20 to 50r/s burst 500. The module-eval case reads the option values. --- docs/swarm/services.md | 16 ++++++++-------- nix/host-modules/swarm-victorialogs.nix | 12 +++++++----- nix/module-eval/core-toggle.nix | 5 +++-- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 59710405..5a2385cd 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -184,14 +184,14 @@ Drilldown_ app is deliberately not installed: it only supports Loki, and no setting here changes that, so Explore is the log browser for this swarm. -| Option | When you'd touch it | -| ------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `deploy.victorialogs.retentionPeriod` | Default `30d`, far shorter than the metrics store's — logs are bulkier per unit of value and are typically read within days of ingestion. Raise it if you need to answer questions about last quarter. | -| `deploy.victorialogs.authRate` | Default `5r/s` per client address, for agent queries checked by `swarm-logs-auth`; over it and `authBurst`, the gateway answers 403. Raise it if many agents share one address and query at once. | -| `deploy.victorialogs.authBurst` | Default `20` requests above `authRate` before the 403s start. | -| `deploy.victorialogs.authVerdictTtl` | Default `60` seconds that `swarm-logs-auth` keeps granting an agent secret it verified, without asking the secret store. A replaced or deleted secret keeps working at the log store for up to that long. | -| `swarm.victorialogs.domain` | Only to rename it. | -| `swarm.victorialogs.port` | Only if something else on the services host already claims `9428`. | +| Option | When you'd touch it | +| ------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `deploy.victorialogs.retentionPeriod` | Default `30d`, far shorter than the metrics store's — logs are bulkier per unit of value and are typically read within days of ingestion. Raise it if you need to answer questions about last quarter. | +| `deploy.victorialogs.authRate` | Default `50r/s` per client address, for agent queries checked by `swarm-logs-auth`; over it and `authBurst`, the gateway answers 403. Agents on one host share the address; the default covers about a hundred cold-starting together. | +| `deploy.victorialogs.authBurst` | Default `500` requests above `authRate` before the 403s start. | +| `deploy.victorialogs.authVerdictTtl` | Default `60` seconds that `swarm-logs-auth` keeps granting an agent secret it verified, without asking the secret store. A replaced or deleted secret keeps working at the log store for up to that long. | +| `swarm.victorialogs.domain` | Only to rename it. | +| `swarm.victorialogs.port` | Only if something else on the services host already claims `9428`. | Like the metrics store, it binds loopback and takes no credential of its own: the gateway vhost is the only way in, and the collector is the only diff --git a/nix/host-modules/swarm-victorialogs.nix b/nix/host-modules/swarm-victorialogs.nix index b3a5149e..6d9e4f82 100644 --- a/nix/host-modules/swarm-victorialogs.nix +++ b/nix/host-modules/swarm-victorialogs.nix @@ -157,7 +157,7 @@ in options.services.hyperhive.deploy.victorialogs.authRate = lib.mkOption { type = lib.types.str; - default = "5r/s"; + default = "50r/s"; example = "30r/m"; description = '' nginx `limit_req` rate, per client address, of `/select/logsql/` @@ -166,15 +166,17 @@ in within {option}`services.hyperhive.deploy.victorialogs.authVerdictTtl` costs a secret store login and read, whether the bearer is right or not. - Counts every agent query, granted or not, so it is sized for the agents - that share one address querying at once, not for one agent. Requests - over the rate and its burst are answered 403. + Counts every agent query, granted or not, and is keyed by client address, + so every agent on one host shares one budget. The default covers about a + hundred agents on a host cold-starting at once, each costing one store + login and read for its first query. Requests over the rate and its burst + are answered 403. ''; }; options.services.hyperhive.deploy.victorialogs.authBurst = lib.mkOption { type = lib.types.ints.positive; - default = 20; + default = 500; description = '' Requests a client address may make above {option}`services.hyperhive.deploy.victorialogs.authRate` before nginx diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index 8cd6b1a3..26a7bd99 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -555,10 +555,11 @@ let l = allLocal.services.nginx.virtualHosts."logs.t.local".locations; http = allLocal.services.nginx.appendHttpConfig; unit = allLocal.systemd.services.swarm-logs-auth; + v = allLocal.services.hyperhive.deploy.victorialogs; in - lib.hasInfix "limit_req_zone $binary_remote_addr zone=swarm_logs_agent_auth:1m rate=5r/s;" http + lib.hasInfix "limit_req_zone $binary_remote_addr zone=swarm_logs_agent_auth:1m rate=${v.authRate};" http && - lib.hasInfix "limit_req zone=swarm_logs_agent_auth burst=20 nodelay;" + lib.hasInfix "limit_req zone=swarm_logs_agent_auth burst=${toString v.authBurst} nodelay;" l."= /__swarm_logs_agent_auth".extraConfig && lib.hasInfix "limit_req_status 403;" l."= /__swarm_logs_agent_auth".extraConfig && !(lib.hasInfix "limit_req" l."^~ /select/logsql/".extraConfig)