Watch
0
0
Fork
You've already forked hyperhive
0

swarm-victorialogs: own network namespace, gateway over a unix socket

This commit is contained in:
maia 2026-11-18 19:12:53 +01:00 • committed by mara
commit 188dbcea8b
2 changed files with 155 additions and 76 deletions

View file

@ -85,6 +85,16 @@ let
in
{
imports = [
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "victorialogs" "port" ] ''
VictoriaLogs listens on a unix socket in a network namespace of its own,
and the gateway reaches it there, so it binds no host port. Remove this
definition.
'')
(lib.mkRemovedOptionModule [ "services" "hyperhive" "deploy" "victorialogs" "port" ] ''
VictoriaLogs listens on a unix socket in a network namespace of its own,
and the gateway reaches it there, so it binds no host port. Remove this
definition.
'')
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "victoriametrics" "port" ] ''
VictoriaMetrics listens on a unix socket in a network namespace of its
own, and the gateway reaches it there, so it binds no host port. Remove
@ -517,10 +527,6 @@ in
[ "services" "hyperhive" "swarm" "otel" "telemetryPort" ]
[ "services" "hyperhive" "deploy" "swarm-otel" "telemetryPort" ]
)
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "victorialogs" "port" ]
[ "services" "hyperhive" "deploy" "victorialogs" "port" ]
)
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "forge" "httpPort" ]
[ "services" "hyperhive" "deploy" "forgejo" "httpPort" ]

View file

@ -17,8 +17,8 @@
# below, same shape as `swarm-ui.nix`'s) — see that file's copy for the full
# rationale (forceSSL is load-bearing there too: authelia answers a plain-http
# auth subrequest with 400, which `auth_request` cannot interpret as anything
# but a broken check). The store's own listener stays loopback-only and
# unauthenticated, and the vhost is now the ONLY way in from outside: the
# but a broken check). The store's own listener is a unix socket in a
# netns of its own and is unauthenticated, and the vhost is the ONLY way in: the
# collector pushes through it too, at its own `= /insert/...` location, because
# a swarm has one log store and the collector need not share a host with it.
# ⚠️ That ingest location deliberately does not carry `swarmAuthRequest` — a
@ -100,9 +100,23 @@ let
grafanaAuthCredential = id: "/run/credentials/${grafanaAuthUnit}.service/${id}";
grafanaBearerStorePath = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer";
# Shared host netns, like every sibling swarm container: Grafana reaches
# this at 127.0.0.1:<port>.
privateNetwork = false;
# A netns of its own on the hive bridge, as the cache's container has
# (./swarm-attic.nix): the forwarder below dials the swarm collector, authelia
# and the store by name, so the container keeps an interface and the bridge's
# resolver. The store listens on a unix socket only, so no host port is
# bound. Grafana reaches the store by its domain through the gateway.
privateNetwork = true;
# The unix socket the gateway's nginx connects to. The directory's mode is
# what keeps it private, so it holds this one socket.
socketDir = "/run/swarm-victorialogs";
socketPath = "${socketDir}/victorialogs.sock";
socketGroup = "gateway-nginx";
# Only the number crosses into the container, which knows the group by name.
nginxGid = config.ids.gids.nginx;
# The upstream every location below forwards to. A location that adds a path
# writes `<upstream>:<path>`; one with no path leaves the request's own.
upstream = "http://unix:${socketPath}";
# The container's journal forwarder pushes to its own receiver on the swarm
# collector, the route `serviceProducers.swarm-victorialogs` in
@ -110,8 +124,7 @@ let
forwarderClientId = cfg.otel.clientId;
forwarderEndpoint = "https://${hyperhiveCfg.swarm.otel.domain}/${cfg.machine}";
forwarderUnit = "swarm-journal-forwarder-oidc";
# In the host netns: distinct from every other collector's there, which the
# assertion below checks.
# Loopback, in the container's own netns.
forwarderTelemetryPort = 8897;
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
# The responder's store identity, handed into the container as system
@ -134,7 +147,7 @@ in
{
# What the store IS from any hive's point of view, the name it answers on, is
# `swarm.victorialogs` in ./swarm-victorialogs-service.nix.
# `enable`, `package`, `port` and `retentionPeriod` are decisions of the host that
# `enable`, `package` and `retentionPeriod` are decisions of the host that
# runs it and live under `deploy.*`.
# Retention is a property of the store this host runs, not something the
@ -149,22 +162,6 @@ in
description = "VictoriaLogs package to run.";
};
options.services.hyperhive.deploy.victorialogs.port = lib.mkOption {
type = lib.types.port;
default = 9428;
description = ''
Loopback port the store listens on. Upstream's default, kept
because there is no reason to move it and a familiar number is
one less thing an operator has to look up.
⚠️ Every swarm container shares the host's network namespace, so
this is a swarm-wide claim rather than a per-container one — two
modules picking the same number collide at runtime with no bind
error and nothing in any log. No repo-wide
check catches it.
'';
};
options.services.hyperhive.deploy.victorialogs.retentionPeriod = lib.mkOption {
type = lib.types.str;
default = "30d";
@ -310,20 +307,6 @@ in
Declare it in swarm-otel-service.nix.
'';
}
{
assertion =
!(lib.elem forwarderTelemetryPort [
hyperhiveCfg.otel.telemetryPort
hyperhiveCfg.deploy.bao.otel.telemetryPort
hyperhiveCfg.deploy.nats.otel.telemetryPort
]);
message = ''
The log store's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
collides with another collector's in the host netns: the hive
collector's, the store forwarder's or the
queue forwarder's.
'';
}
];
# nspawn fails to start the container when a credential source is
@ -375,9 +358,9 @@ in
services.hyperhive.gateway.enable = lib.mkDefault true;
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
# Authenticated front door onto the loopback-only store — see the
# Authenticated front door onto the store's socket — see the
# file-top comment for why this is safe to add without touching the
# store's own (still unauthenticated, still loopback) listener at all.
# store's own (still unauthenticated, socket-only) listener at all.
# `forceSSL`: same asymmetry `swarm-ui.nix` documents —
# authelia answers a plain-http auth subrequest with 400, which
# `auth_request` cannot read as anything but a broken check, so this
@ -387,7 +370,7 @@ in
forceSSL = true;
locations = {
"/" = {
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/";
proxyPass = "${upstream}:/";
extraConfig = swarmAuthRequest;
};
@ -406,7 +389,7 @@ in
# note on `swarmAuthRequest` above), so naming it here is required
# rather than redundant.
"= /insert/opentelemetry/v1/logs" = {
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/insert/opentelemetry/v1/logs";
proxyPass = "${upstream}:/insert/opentelemetry/v1/logs";
extraConfig = ''
auth_request /__swarm_logs_ingest_authz;
'';
@ -440,7 +423,7 @@ in
"^~ /select/logsql/" = {
# No URI part, so the request path and its query string reach
# the store as the caller sent them.
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}";
proxyPass = upstream;
extraConfig = ''
auth_request /__swarm_logs_auth;
'';
@ -457,7 +440,7 @@ in
# authelia's rules for this URL admit the collector's client and
# deny every other subject (see `metricsRules` in ./swarm-authelia.nix).
"= /metrics" = {
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/metrics";
proxyPass = "${upstream}:/metrics";
extraConfig = ''
auth_request /__hive_authelia;
'';
@ -659,6 +642,12 @@ in
};
};
# The socket directory is the gateway's way in: its group is nginx's.
systemd.tmpfiles.rules = [ "d ${socketDir} 0770 root ${toString nginxGid} - -" ];
# The hive bridge carries the container's DHCP lease and its resolver.
services.hyperhive.network.enable = lib.mkDefault true;
containers.${cfg.machine} = {
autoStart = true;
ephemeral = false;
@ -668,10 +657,20 @@ in
id: source: "--load-credential=${id}:${source}"
) forwarderStoreFiles;
inherit privateNetwork;
bindMounts = caTrust.bindMount;
hostBridge = networkCfg.bridgeName;
# A bind mount, not a path both sides know: a container's `/run` is an
# nspawn tmpfs the host cannot see into. Connecting to a socket is a
# write to it.
bindMounts = {
${socketDir} = {
hostPath = socketDir;
isReadOnly = false;
};
}
// caTrust.bindMount;
config =
{ ... }:
{ pkgs, ... }:
{
imports = [
../container-modules/swarm-container.nix
@ -697,6 +696,37 @@ in
services.hyperhive.swarmContainer = { inherit privateNetwork; };
# The bridge address comes from the gateway's DHCP. `swarm-bridge-dns`
# owns resolv.conf, so dhcpcd must not write it over that unit.
networking.interfaces.eth0.useDHCP = true;
networking.dhcpcd.extraConfig = "nohook resolv.conf";
# The socket's group is the store's primary group, so that group is
# nginx's gid. A static user, not nixpkgs' DynamicUser: a dynamic
# user's group is allocated per start, and nginx is never in it.
users.groups.${socketGroup}.gid = nginxGid;
users.users.victorialogs = {
isSystemUser = true;
group = socketGroup;
};
systemd.services.victorialogs = {
serviceConfig = {
DynamicUser = lib.mkForce false;
User = "victorialogs";
Group = socketGroup;
# The store creates its socket under the process umask, so `0007`
# makes it `0770` and nginx's group can connect.
UMask = "0007";
};
# nixpkgs' readiness loop curls a TCP URL, which no `unix:` address
# answers.
postStart = lib.mkForce ''
until ${lib.getExe pkgs.curl} -s -o /dev/null --unix-socket ${socketPath} http://localhost/ping; do
sleep 1;
done
'';
};
# Authenticated as the forwarder's own OIDC client, with the
# responder's store identity: one identity for the whole container.
# It is a system credential, so the fetch unit loads it.
@ -722,12 +752,13 @@ in
enable = true;
package = deployCfg.victorialogs.package;
# ⚠️ PINNED TO LOOPBACK: upstream's default listens on every
# interface, and this endpoint accepts writes as well as reads.
# Nothing authenticates it either — upstream offers one basic-auth
# pair (`-httpAuth.username` / `-httpAuth.password`) and this
# module sets neither. The bind address is the boundary.
listenAddress = "127.0.0.1:${toString deployCfg.victorialogs.port}";
# ⚠️ A SOCKET, not upstream's default of every interface: this
# endpoint accepts writes as well as reads. Nothing authenticates it
# either — upstream offers one basic-auth pair
# (`-httpAuth.username` / `-httpAuth.password`) and this module sets
# neither. The socket's directory mode is the boundary: only the
# gateway's group can connect.
listenAddress = "unix:${socketPath}";
extraOptions = [ "-retentionPeriod=${deployCfg.victorialogs.retentionPeriod}" ];
};
@ -749,7 +780,7 @@ in
# code is not evidence.
#
# ⚠️ Like the metrics store's, that endpoint is unauthenticated — which is
# why `listenAddress` above is loopback and why nothing reaches it except
# why `listenAddress` above is a socket and why nothing reaches it except
# through the gateway, where the ingest route is authenticated. (This module
# declares a vhost.)
}