vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-logs-auth: check agent tokens on the log store's query route

Browse source 
A new `auth_request` responder on the log store's host. nginx sends it
every `/select/logsql/` request whose `Authorization` is
`Bearer swarm-agent.<agent>.<secret>`; it answers 200 when the secret is
the one stored at `swarm/agents/<agent>/queue` (`swarm-agent-verify`),
401 otherwise. Every other bearer still goes to authelia, so agents
still presenting the hive's shared OIDC client keep working.

`auth_request` takes a fixed URI, so the split is a `map` on
`$http_authorization` plus an internal location that rewrites to the
map's pick. Checked against nginx 1.30.5 with stub upstreams: an agent
token only the stub validator accepts gets 200, a token only the stub
authelia accepts gets 200, every wrong one 401, and the internal
locations 404 from outside.

The responder serves a unix socket in its own traverse-only runtime
directory and reads the store under its own leaf
(`swarm-logs-auth`), whose policy is `swarm/agents/+/queue`, read-only.
glue-bao-tls mints the leaf where the store runs; elsewhere the
operator places it, and eval refuses a log store host without one.

Part of #4459.
This commit is contained in:
atlas 2026-10-12 22:48:19 +02:00 • committed by mara
commit 1f4736e79b
16 changed files with 515 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

155
nix/host-modules/swarm-victorialogs.nix
View file

@ -52,6 +52,22 @@ let
error_page 401 =302 https://${hyperhiveCfg.swarm.authelia.domain}/?rd=$target_url;
'';
# The agents' `auth_request` responder (`swarm-logs-auth`). Its socket's
# directory holds that socket alone and is traverse-only for others, the
# shape `swarm-controller.nix` gives its own.
logsAuthUnit = "swarm-logs-auth";
logsAuthSocket = "/run/${logsAuthUnit}/auth.sock";
logsAuthCredential = id: "/run/credentials/${logsAuthUnit}.service/${id}";
# Credential id → host path of the responder's store identity.
logsAuthStoreFiles = {
"bao-client.pem" = deployCfg.victorialogs.authBaoClientCertFile;
"bao-client-key.pem" = deployCfg.victorialogs.authBaoClientKeyFile;
}
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
"bao-ca.pem" = deployCfg.bao.serverCaFile;
};
baoCfg = hyperhiveCfg.swarm.bao;
# Shared host netns, like every sibling swarm container: the collector
# and Grafana reach this at 127.0.0.1:<port>.
privateNetwork = false;
@ -88,7 +104,74 @@ in
'';
};
options.services.hyperhive.deploy.victorialogs.authPackage = lib.mkOption {
type = lib.types.package;
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-logs-auth";
description = ''
The `swarm-logs-auth` package: the gateway's `auth_request` responder
that checks an agent's `swarm-agent.<agent>.<secret>` bearer on
`/select/logsql/` against the secret store.
'';
};
options.services.hyperhive.deploy.victorialogs.authBaoClientCertFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "/var/lib/swarm-bao-pki/logs-auth.pem";
description = ''
Client certificate `swarm-logs-auth` presents to the swarm's secret store
to read agents' queue secrets. Its subject must be
{option}`services.hyperhive.deploy.bao.logsAuthCommonName`.
Required when `deploy.victorialogs.enable` is set; unset fails at eval.
./glue-logs-auth-bao-identity.nix points it at the leaf
./glue-bao-tls.nix mints on a host that runs the store; on any other
log-store host it is a file the operator places.
A path, never a value.
'';
};
options.services.hyperhive.deploy.victorialogs.authBaoClientKeyFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "/var/lib/swarm-bao-pki/logs-auth-key.pem";
description = ''
Private key for {option}`services.hyperhive.deploy.victorialogs.authBaoClientCertFile`.
A path, never a value.
'';
};
config = lib.mkIf deployCfg.victorialogs.enable {
assertions = [
{
assertion =
deployCfg.victorialogs.authBaoClientCertFile != null
&& deployCfg.victorialogs.authBaoClientKeyFile != null;
message = ''
services.hyperhive.deploy.victorialogs.authBaoClientCertFile and
authBaoClientKeyFile must both be set.
swarm-logs-auth reads agents' queue secrets from the swarm's secret
store with this certificate. They default to the leaf the store
mints where services.hyperhive.deploy.bao.enable is set on this
host; elsewhere, issue a leaf whose CN is
services.hyperhive.deploy.bao.logsAuthCommonName and name it in
both options.
'';
}
];
# Which check a `/select/logsql/` request gets. `auth_request` takes a
# fixed URI, so it names `/__swarm_logs_auth`, which rewrites to the
# target this map picks from the caller's `Authorization`.
services.nginx.appendHttpConfig = ''
map $http_authorization $swarm_logs_auth_target {
"~^Bearer swarm-agent\." /__swarm_logs_agent_auth;
default /__hive_authelia;
}
'';
# This store publishes its own health as prometheus metrics on the same
# listener it serves queries on, so the swarm's collector scrapes it with
# no exporter and no extra port — same arrangement as the metrics store.
@ -174,12 +257,35 @@ in
# logs. That is the rule in force, not an omission: read
# permissions are a later thing, and this location is where one
# attaches when it exists.
#
# Two checks: an agent's own `swarm-agent.` bearer goes to
# `swarm-logs-auth`, anything else to authelia (`appendHttpConfig`
# above picks).
"^~ /select/logsql/" = {
# No URI part, so the request path and its query string reach
# the store as the caller sent them.
proxyPass = "http://127.0.0.1:${toString cfg.port}";
extraConfig = ''
auth_request /__hive_authelia;
auth_request /__swarm_logs_auth;
'';
};
# `rewrite … last` re-matches locations, so the subrequest lands in
# one of the two internal locations named by the map.
"= /__swarm_logs_auth" = {
extraConfig = ''
internal;
rewrite ^ $swarm_logs_auth_target last;
'';
};
# The `Authorization` header rides the subrequest unchanged.
"= /__swarm_logs_agent_auth" = {
proxyPass = "http://unix:${logsAuthSocket}:/";
extraConfig = ''
internal;
proxy_pass_request_body off;
proxy_set_header Content-Length "";
'';
};
@ -207,6 +313,53 @@ in
};
};
# The agents' check on `/select/logsql/`. On the host rather than in the
# store's container: it needs nothing of the store, only nginx's reach to
# its socket and its own store identity.
systemd.services.${logsAuthUnit} = {
description = "swarm log store auth_request responder for agent credentials";
wantedBy = [ "multi-user.target" ];
# Where the store's PKI is minted on this host, the leaf is one of its
# files. Ordering only: elsewhere the unit does not exist.
after = [
"network.target"
"swarm-bao-pki.service"
];
serviceConfig = {
ExecStart = lib.concatStringsSep " " [
"${deployCfg.victorialogs.authPackage}/bin/swarm-logs-auth"
"--socket ${logsAuthSocket}"
# The role ./swarm-bao.nix writes on the store's `cert` mount.
"--store-cert-role swarm-logs-auth"
];
# Paths only: `argv` is world-readable via /proc/<pid>/cmdline.
LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") logsAuthStoreFiles;
DynamicUser = true;
RuntimeDirectory = logsAuthUnit;
# Traverse-only for others, so nginx reaches the socket path.
RuntimeDirectoryMode = "0751";
# The path never vanishes from under a running nginx; the responder
# unlinks a stale socket as it starts.
RuntimeDirectoryPreserve = "yes";
Restart = "on-failure";
RestartSec = "5s";
SyslogIdentifier = logsAuthUnit;
ProtectSystem = "strict";
ProtectHome = true;
PrivateTmp = true;
PrivateDevices = true;
NoNewPrivileges = true;
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = logsAuthCredential "bao-client.pem";
BAO_CLIENT_KEY = logsAuthCredential "bao-client-key.pem";
}
// lib.optionalAttrs (logsAuthStoreFiles ? "bao-ca.pem") {
BAO_CACERT = logsAuthCredential "bao-ca.pem";
};
};
containers.${cfg.machine} = {
autoStart = true;
ephemeral = false;