Watch
0
0
Fork
You've already forked hyperhive
0

swarm-victorialogs: own network namespace, gateway over a unix socket

This commit is contained in:
maia 2026-11-18 19:12:53 +01:00 • committed by mara
commit 188dbcea8b
5 changed files with 155 additions and 76 deletions

View file

@ -85,6 +85,16 @@ let
in
{
imports = [
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "victorialogs" "port" ] ''
VictoriaLogs listens on a unix socket in a network namespace of its own,
and the gateway reaches it there, so it binds no host port. Remove this
definition.
'')
(lib.mkRemovedOptionModule [ "services" "hyperhive" "deploy" "victorialogs" "port" ] ''
VictoriaLogs listens on a unix socket in a network namespace of its own,
and the gateway reaches it there, so it binds no host port. Remove this
definition.
'')
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "victoriametrics" "port" ] ''
VictoriaMetrics listens on a unix socket in a network namespace of its
own, and the gateway reaches it there, so it binds no host port. Remove
@ -517,10 +527,6 @@ in
[ "services" "hyperhive" "swarm" "otel" "telemetryPort" ]
[ "services" "hyperhive" "deploy" "swarm-otel" "telemetryPort" ]
)
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "victorialogs" "port" ]
[ "services" "hyperhive" "deploy" "victorialogs" "port" ]
)
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "forge" "httpPort" ]
[ "services" "hyperhive" "deploy" "forgejo" "httpPort" ]

View file

@ -17,8 +17,8 @@
# below, same shape as `swarm-ui.nix`'s) — see that file's copy for the full
# rationale (forceSSL is load-bearing there too: authelia answers a plain-http
# auth subrequest with 400, which `auth_request` cannot interpret as anything
# but a broken check). The store's own listener stays loopback-only and
# unauthenticated, and the vhost is now the ONLY way in from outside: the
# but a broken check). The store's own listener is a unix socket in a
# netns of its own and is unauthenticated, and the vhost is the ONLY way in: the
# collector pushes through it too, at its own `= /insert/...` location, because
# a swarm has one log store and the collector need not share a host with it.
# ⚠️ That ingest location deliberately does not carry `swarmAuthRequest` — a
@ -100,9 +100,23 @@ let
grafanaAuthCredential = id: "/run/credentials/${grafanaAuthUnit}.service/${id}";
grafanaBearerStorePath = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer";
# Shared host netns, like every sibling swarm container: Grafana reaches
# this at 127.0.0.1:<port>.
privateNetwork = false;
# A netns of its own on the hive bridge, as the cache's container has
# (./swarm-attic.nix): the forwarder below dials the swarm collector, authelia
# and the store by name, so the container keeps an interface and the bridge's
# resolver. The store listens on a unix socket only, so no host port is
# bound. Grafana reaches the store by its domain through the gateway.
privateNetwork = true;
# The unix socket the gateway's nginx connects to. The directory's mode is
# what keeps it private, so it holds this one socket.
socketDir = "/run/swarm-victorialogs";
socketPath = "${socketDir}/victorialogs.sock";
socketGroup = "gateway-nginx";
# Only the number crosses into the container, which knows the group by name.
nginxGid = config.ids.gids.nginx;
# The upstream every location below forwards to. A location that adds a path
# writes `<upstream>:<path>`; one with no path leaves the request's own.
upstream = "http://unix:${socketPath}";
# The container's journal forwarder pushes to its own receiver on the swarm
# collector, the route `serviceProducers.swarm-victorialogs` in
@ -110,8 +124,7 @@ let
forwarderClientId = cfg.otel.clientId;
forwarderEndpoint = "https://${hyperhiveCfg.swarm.otel.domain}/${cfg.machine}";
forwarderUnit = "swarm-journal-forwarder-oidc";
# In the host netns: distinct from every other collector's there, which the
# assertion below checks.
# Loopback, in the container's own netns.
forwarderTelemetryPort = 8897;
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
# The responder's store identity, handed into the container as system
@ -134,7 +147,7 @@ in
{
# What the store IS from any hive's point of view, the name it answers on, is
# `swarm.victorialogs` in ./swarm-victorialogs-service.nix.
# `enable`, `package`, `port` and `retentionPeriod` are decisions of the host that
# `enable`, `package` and `retentionPeriod` are decisions of the host that
# runs it and live under `deploy.*`.
# Retention is a property of the store this host runs, not something the
@ -149,22 +162,6 @@ in
description = "VictoriaLogs package to run.";
};
options.services.hyperhive.deploy.victorialogs.port = lib.mkOption {
type = lib.types.port;
default = 9428;
description = ''
Loopback port the store listens on. Upstream's default, kept
because there is no reason to move it and a familiar number is
one less thing an operator has to look up.
⚠️ Every swarm container shares the host's network namespace, so
this is a swarm-wide claim rather than a per-container one — two
modules picking the same number collide at runtime with no bind
error and nothing in any log. No repo-wide
check catches it.
'';
};
options.services.hyperhive.deploy.victorialogs.retentionPeriod = lib.mkOption {
type = lib.types.str;
default = "30d";
@ -310,20 +307,6 @@ in
Declare it in swarm-otel-service.nix.
'';
}
{
assertion =
!(lib.elem forwarderTelemetryPort [
hyperhiveCfg.otel.telemetryPort
hyperhiveCfg.deploy.bao.otel.telemetryPort
hyperhiveCfg.deploy.nats.otel.telemetryPort
]);
message = ''
The log store's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
collides with another collector's in the host netns: the hive
collector's, the store forwarder's or the
queue forwarder's.
'';
}
];
# nspawn fails to start the container when a credential source is
@ -375,9 +358,9 @@ in
services.hyperhive.gateway.enable = lib.mkDefault true;
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
# Authenticated front door onto the loopback-only store — see the
# Authenticated front door onto the store's socket — see the
# file-top comment for why this is safe to add without touching the
# store's own (still unauthenticated, still loopback) listener at all.
# store's own (still unauthenticated, socket-only) listener at all.
# `forceSSL`: same asymmetry `swarm-ui.nix` documents —
# authelia answers a plain-http auth subrequest with 400, which
# `auth_request` cannot read as anything but a broken check, so this
@ -387,7 +370,7 @@ in
forceSSL = true;
locations = {
"/" = {
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/";
proxyPass = "${upstream}:/";
extraConfig = swarmAuthRequest;
};
@ -406,7 +389,7 @@ in
# note on `swarmAuthRequest` above), so naming it here is required
# rather than redundant.
"= /insert/opentelemetry/v1/logs" = {
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/insert/opentelemetry/v1/logs";
proxyPass = "${upstream}:/insert/opentelemetry/v1/logs";
extraConfig = ''
auth_request /__swarm_logs_ingest_authz;
'';
@ -440,7 +423,7 @@ in
"^~ /select/logsql/" = {
# No URI part, so the request path and its query string reach
# the store as the caller sent them.
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}";
proxyPass = upstream;
extraConfig = ''
auth_request /__swarm_logs_auth;
'';
@ -457,7 +440,7 @@ in
# authelia's rules for this URL admit the collector's client and
# deny every other subject (see `metricsRules` in ./swarm-authelia.nix).
"= /metrics" = {
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/metrics";
proxyPass = "${upstream}:/metrics";
extraConfig = ''
auth_request /__hive_authelia;
'';
@ -659,6 +642,12 @@ in
};
};
# The socket directory is the gateway's way in: its group is nginx's.
systemd.tmpfiles.rules = [ "d ${socketDir} 0770 root ${toString nginxGid} - -" ];
# The hive bridge carries the container's DHCP lease and its resolver.
services.hyperhive.network.enable = lib.mkDefault true;
containers.${cfg.machine} = {
autoStart = true;
ephemeral = false;
@ -668,10 +657,20 @@ in
id: source: "--load-credential=${id}:${source}"
) forwarderStoreFiles;
inherit privateNetwork;
bindMounts = caTrust.bindMount;
hostBridge = networkCfg.bridgeName;
# A bind mount, not a path both sides know: a container's `/run` is an
# nspawn tmpfs the host cannot see into. Connecting to a socket is a
# write to it.
bindMounts = {
${socketDir} = {
hostPath = socketDir;
isReadOnly = false;
};
}
// caTrust.bindMount;
config =
{ ... }:
{ pkgs, ... }:
{
imports = [
../container-modules/swarm-container.nix
@ -697,6 +696,37 @@ in
services.hyperhive.swarmContainer = { inherit privateNetwork; };
# The bridge address comes from the gateway's DHCP. `swarm-bridge-dns`
# owns resolv.conf, so dhcpcd must not write it over that unit.
networking.interfaces.eth0.useDHCP = true;
networking.dhcpcd.extraConfig = "nohook resolv.conf";
# The socket's group is the store's primary group, so that group is
# nginx's gid. A static user, not nixpkgs' DynamicUser: a dynamic
# user's group is allocated per start, and nginx is never in it.
users.groups.${socketGroup}.gid = nginxGid;
users.users.victorialogs = {
isSystemUser = true;
group = socketGroup;
};
systemd.services.victorialogs = {
serviceConfig = {
DynamicUser = lib.mkForce false;
User = "victorialogs";
Group = socketGroup;
# The store creates its socket under the process umask, so `0007`
# makes it `0770` and nginx's group can connect.
UMask = "0007";
};
# nixpkgs' readiness loop curls a TCP URL, which no `unix:` address
# answers.
postStart = lib.mkForce ''
until ${lib.getExe pkgs.curl} -s -o /dev/null --unix-socket ${socketPath} http://localhost/ping; do
sleep 1;
done
'';
};
# Authenticated as the forwarder's own OIDC client, with the
# responder's store identity: one identity for the whole container.
# It is a system credential, so the fetch unit loads it.
@ -722,12 +752,13 @@ in
enable = true;
package = deployCfg.victorialogs.package;
# ⚠️ PINNED TO LOOPBACK: upstream's default listens on every
# interface, and this endpoint accepts writes as well as reads.
# Nothing authenticates it either — upstream offers one basic-auth
# pair (`-httpAuth.username` / `-httpAuth.password`) and this
# module sets neither. The bind address is the boundary.
listenAddress = "127.0.0.1:${toString deployCfg.victorialogs.port}";
# ⚠️ A SOCKET, not upstream's default of every interface: this
# endpoint accepts writes as well as reads. Nothing authenticates it
# either — upstream offers one basic-auth pair
# (`-httpAuth.username` / `-httpAuth.password`) and this module sets
# neither. The socket's directory mode is the boundary: only the
# gateway's group can connect.
listenAddress = "unix:${socketPath}";
extraOptions = [ "-retentionPeriod=${deployCfg.victorialogs.retentionPeriod}" ];
};
@ -749,7 +780,7 @@ in
# code is not evidence.
#
# ⚠️ Like the metrics store's, that endpoint is unauthenticated — which is
# why `listenAddress` above is loopback and why nothing reaches it except
# why `listenAddress` above is a socket and why nothing reaches it except
# through the gateway, where the ingest route is authenticated. (This module
# declares a vhost.)
}

View file

@ -2262,10 +2262,10 @@ let
{
# Each old spelling lands on the new path, with its own rename warning,
# and its value reaches the unit or vhost that reads it. Ports are chosen
# so three of them collide on purpose: the bao collector's telemetry
# port equals the NATS monitor port and the forge forwarder's, and the
# NATS forwarder's equals the log store forwarder's. Each of the three
# colocation assertions must read the moved options to see it.
# so two of them collide on purpose: the bao collector's telemetry
# port equals the NATS monitor port and the forge forwarder's. Each of the
# two colocation assertions must read the moved options to see it. (The log
# store has a netns of its own, so its forwarder's port collides with none.)
name = "the per-service settings moved to deploy.* still apply from their old paths, with a rename warning each";
ok =
let
@ -2285,7 +2285,6 @@ let
nats.metricsPort = 17777;
nats.otel.telemetryPort = 8897;
otel.telemetryPort = 18889;
victorialogs.port = 19428;
forge.httpPort = 13000;
matrix.allowEncryption = true;
controller.links = [
@ -2322,7 +2321,6 @@ let
&& d.nats.metricsPort == 17777
&& d.nats.otel.telemetryPort == 8897
&& d.swarm-otel.telemetryPort == 18889
&& d.victorialogs.port == 19428
&& d.forgejo.httpPort == 13000
&& d.matrix.allowEncryption
&& d.builder.nixpkgsFlake == "path:/legacy/nixpkgs"
@ -2343,7 +2341,6 @@ let
old.containers.swarm-builder.config.systemd.services.swarm-builder.environment.SWARM_BUILDER_NIXPKGS_FLAKE
== "path:/legacy/nixpkgs"
&& (locs sw.nats.domain)."= /metrics".proxyPass == "http://127.0.0.1:17777/metrics"
&& (locs sw.victorialogs.domain)."= /metrics".proxyPass == "http://127.0.0.1:19428/metrics"
&& (locs sw.forge.domain)."/".proxyPass == "http://127.0.0.1:13000/"
&& old.containers.hive-matrix.config.services.matrix-tuwunel.settings.global.allow_encryption
&& lib.all warned [
@ -2354,7 +2351,6 @@ let
"swarm.nats.metricsPort"
"swarm.nats.otel.telemetryPort"
"swarm.otel.telemetryPort"
"swarm.victorialogs.port"
"swarm.forge.httpPort"
"swarm.matrix.allowEncryption"
"c0re.nixpkgsFlake"
@ -2365,10 +2361,8 @@ let
&& !(lib.any (w: lib.hasInfix "swarm.controller.queue" w) allLocal.warnings)
&& refused old "deploy.nats.otel.telemetryPort"
&& refused old "The forge's journal forwarder's telemetry port"
&& refused old "The log store's journal forwarder's telemetry port"
&& !(refused allLocal "deploy.nats.otel.telemetryPort")
&& !(refused allLocal "The forge's journal forwarder's telemetry port")
&& !(refused allLocal "The log store's journal forwarder's telemetry port");
&& !(refused allLocal "The forge's journal forwarder's telemetry port");
}
{
# The default render of the moved options, pinned so moving them changes
@ -2389,7 +2383,9 @@ let
&& h.deploy.nats.otel.telemetryPort == 8891
&& h.deploy.swarm-otel.telemetryPort == 8889
&& (locs h.swarm.nats.domain)."= /metrics".proxyPass == "http://127.0.0.1:7777/metrics"
&& (locs h.swarm.victorialogs.domain)."= /metrics".proxyPass == "http://127.0.0.1:9428/metrics"
&&
(locs h.swarm.victorialogs.domain)."= /metrics".proxyPass
== "http://unix:/run/swarm-victorialogs/victorialogs.sock:/metrics"
&& (locs h.swarm.forge.domain)."/".proxyPass == "http://127.0.0.1:3000/"
&& !allLocal.containers.hive-matrix.config.services.matrix-tuwunel.settings.global.allow_encryption
&& env.SWARM_CONTROLLER_MATRIX_ADMIN_SYNC_IGNORE_USERS == ""

View file

@ -511,7 +511,7 @@ let
in
h.deploy.victorialogs.enable
&& locs ? "= /metrics"
&& locs."= /metrics".proxyPass == "http://127.0.0.1:${toString h.deploy.victorialogs.port}/metrics"
&& locs."= /metrics".proxyPass == "http://unix:/run/swarm-victorialogs/victorialogs.sock:/metrics"
&& checksOf locs."= /metrics" == [ "auth_request /__hive_authelia;" ]
&& !(lib.hasInfix "error_page" locs."= /metrics".extraConfig)
&& lib.hasInfix "error_page 401 =302" locs."/".extraConfig

View file

@ -843,14 +843,14 @@ let
}
{
# Both values come from ../container-modules/swarm-container.nix.
# Four of the 16 service containers share the host netns (hive-forge, bao,
# the queue, the log store) and twelve have one of their own.
# Three of the 16 service containers share the host netns (hive-forge, bao,
# the queue) and thirteen have one of their own.
name = "a service container on the host netns runs no firewall or resolvconf of its own";
ok =
let
shared = lib.filter (c: !c.privateNetwork) (lib.attrValues serviceContainers);
in
lib.length shared == 4
lib.length shared == 3
&& lib.all (
c: !c.config.networking.firewall.enable && !c.config.networking.resolvconf.enable
) shared;
@ -990,6 +990,50 @@ let
&& bridge.serviceConfig.Group == "gateway-nginx"
&& lib.elem dir unit.serviceConfig.ReadWritePaths;
}
{
# On the bridge by DHCP, listening on a socket only the gateway's group can
# reach, so the store binds no host port. Every location that forwards to
# it goes through the socket (the authelia subrequests are the `https://`
# upstream); the read, ingest, scrape and catch-all locations are named so
# an empty filter can't pass, and the old loopback port is named nowhere.
name = "the log store runs on a netns of its own and the gateway reaches it over its socket";
ok =
let
host = serviceContainersWithCi;
net = host.services.hyperhive.network;
c = host.containers.swarm-victorialogs;
dir = "/run/swarm-victorialogs";
sock = "${dir}/victorialogs.sock";
viaSock = path: "http://unix:${sock}:${path}";
locs =
host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.victorialogs.domain}.locations;
toStore = lib.filter (
p: p != null && !(lib.hasPrefix "https://" p) && !(lib.hasPrefix "http://unix:/run/swarm-logs" p)
) (map (l: l.proxyPass) (lib.attrValues locs));
unit = c.config.systemd.services.victorialogs;
nginxGid = host.ids.gids.nginx;
in
c.privateNetwork
&& c.hostBridge == net.bridgeName
&& c.hostAddress == null
&& c.localAddress == null
&& c.config.networking.interfaces.eth0.useDHCP
&& c.config.services.victorialogs.listenAddress == "unix:${sock}"
&& locs."/".proxyPass == viaSock "/"
&& locs."= /insert/opentelemetry/v1/logs".proxyPass == viaSock "/insert/opentelemetry/v1/logs"
&& locs."^~ /select/logsql/".proxyPass == "http://unix:${sock}"
&& locs."= /metrics".proxyPass == viaSock "/metrics"
&& toStore != [ ]
&& lib.all (lib.hasPrefix "http://unix:${sock}") toStore
&& !(lib.any (lib.hasInfix "127.0.0.1") toStore)
&& c.bindMounts.${dir}.hostPath == dir
&& builtins.elem "d ${dir} 0770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules
&& c.config.users.groups.gateway-nginx.gid == nginxGid
&& unit.serviceConfig.Group == "gateway-nginx"
&& !unit.serviceConfig.DynamicUser
&& unit.serviceConfig.UMask == "0007"
&& lib.hasInfix "--unix-socket ${sock}" unit.postStart;
}
{
# On the bridge by DHCP, listening on a socket (protocol `socket` ignores the
# upstream-default `http_port`, which stays in the settings): the