Watch
0
0
Fork
You've already forked hyperhive
0

swarm-victorialogs: own network namespace, gateway over a unix socket

This commit is contained in:
maia 2026-11-18 19:12:53 +01:00 • committed by mara
commit 188dbcea8b
6 changed files with 155 additions and 76 deletions

View file

@ -188,10 +188,12 @@ swarm.
| `deploy.victorialogs.authVerdictTtl` | Default `60` seconds that `swarm-logs-auth` keeps granting an agent secret it verified, without asking the secret store. A replaced or deleted secret keeps working at the log store for up to that long. A revoked `logs` permission does too. |
| `deploy.victorialogs.grafanaAuthPackage` | The `swarm-metrics-auth` build that runs as `swarm-logs-grafana-auth`. Set on its own: `deploy.victoriametrics.authPackage` doesn't reach it. |
| `swarm.victorialogs.domain` | Only to rename it. |
| `deploy.victorialogs.port` | Only if something else on the services host already claims `9428`. |
It binds loopback and takes no credential of its own: the gateway vhost is
the only way in, and the collector is the only intended writer.
It listens on a unix socket in a network namespace of its own, on the hive
bridge so its journal forwarder can reach the swarm collector, and binds no host
port. Only the gateway's nginx can open the socket. It takes no credential of
its own: the gateway vhost is the only way in, and the collector is the only
intended writer.
**Who may query.** `/select/logsql/` takes one of three credentials. An
agent presents its `swarm-agent.<agent>.<secret>` bearer, checked by

View file

@ -85,6 +85,16 @@ let
in
{
imports = [
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "victorialogs" "port" ] ''
VictoriaLogs listens on a unix socket in a network namespace of its own,
and the gateway reaches it there, so it binds no host port. Remove this
definition.
'')
(lib.mkRemovedOptionModule [ "services" "hyperhive" "deploy" "victorialogs" "port" ] ''
VictoriaLogs listens on a unix socket in a network namespace of its own,
and the gateway reaches it there, so it binds no host port. Remove this
definition.
'')
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "victoriametrics" "port" ] ''
VictoriaMetrics listens on a unix socket in a network namespace of its
own, and the gateway reaches it there, so it binds no host port. Remove
@ -517,10 +527,6 @@ in
[ "services" "hyperhive" "swarm" "otel" "telemetryPort" ]
[ "services" "hyperhive" "deploy" "swarm-otel" "telemetryPort" ]
)
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "victorialogs" "port" ]
[ "services" "hyperhive" "deploy" "victorialogs" "port" ]
)
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "forge" "httpPort" ]
[ "services" "hyperhive" "deploy" "forgejo" "httpPort" ]

View file

@ -17,8 +17,8 @@
# below, same shape as `swarm-ui.nix`'s) — see that file's copy for the full
# rationale (forceSSL is load-bearing there too: authelia answers a plain-http
# auth subrequest with 400, which `auth_request` cannot interpret as anything
# but a broken check). The store's own listener stays loopback-only and
# unauthenticated, and the vhost is now the ONLY way in from outside: the
# but a broken check). The store's own listener is a unix socket in a
# netns of its own and is unauthenticated, and the vhost is the ONLY way in: the
# collector pushes through it too, at its own `= /insert/...` location, because
# a swarm has one log store and the collector need not share a host with it.
# ⚠️ That ingest location deliberately does not carry `swarmAuthRequest` — a
@ -100,9 +100,23 @@ let
grafanaAuthCredential = id: "/run/credentials/${grafanaAuthUnit}.service/${id}";
grafanaBearerStorePath = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer";
# Shared host netns, like every sibling swarm container: Grafana reaches
# this at 127.0.0.1:<port>.
privateNetwork = false;
# A netns of its own on the hive bridge, as the cache's container has
# (./swarm-attic.nix): the forwarder below dials the swarm collector, authelia
# and the store by name, so the container keeps an interface and the bridge's
# resolver. The store listens on a unix socket only, so no host port is
# bound. Grafana reaches the store by its domain through the gateway.
privateNetwork = true;
# The unix socket the gateway's nginx connects to. The directory's mode is
# what keeps it private, so it holds this one socket.
socketDir = "/run/swarm-victorialogs";
socketPath = "${socketDir}/victorialogs.sock";
socketGroup = "gateway-nginx";
# Only the number crosses into the container, which knows the group by name.
nginxGid = config.ids.gids.nginx;
# The upstream every location below forwards to. A location that adds a path
# writes `<upstream>:<path>`; one with no path leaves the request's own.
upstream = "http://unix:${socketPath}";
# The container's journal forwarder pushes to its own receiver on the swarm
# collector, the route `serviceProducers.swarm-victorialogs` in
@ -110,8 +124,7 @@ let
forwarderClientId = cfg.otel.clientId;
forwarderEndpoint = "https://${hyperhiveCfg.swarm.otel.domain}/${cfg.machine}";
forwarderUnit = "swarm-journal-forwarder-oidc";
# In the host netns: distinct from every other collector's there, which the
# assertion below checks.
# Loopback, in the container's own netns.
forwarderTelemetryPort = 8897;
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
# The responder's store identity, handed into the container as system
@ -134,7 +147,7 @@ in
{
# What the store IS from any hive's point of view, the name it answers on, is
# `swarm.victorialogs` in ./swarm-victorialogs-service.nix.
# `enable`, `package`, `port` and `retentionPeriod` are decisions of the host that
# `enable`, `package` and `retentionPeriod` are decisions of the host that
# runs it and live under `deploy.*`.
# Retention is a property of the store this host runs, not something the
@ -149,22 +162,6 @@ in
description = "VictoriaLogs package to run.";
};
options.services.hyperhive.deploy.victorialogs.port = lib.mkOption {
type = lib.types.port;
default = 9428;
description = ''
Loopback port the store listens on. Upstream's default, kept
because there is no reason to move it and a familiar number is
one less thing an operator has to look up.
⚠️ Every swarm container shares the host's network namespace, so
this is a swarm-wide claim rather than a per-container one — two
modules picking the same number collide at runtime with no bind
error and nothing in any log. No repo-wide
check catches it.
'';
};
options.services.hyperhive.deploy.victorialogs.retentionPeriod = lib.mkOption {
type = lib.types.str;
default = "30d";
@ -310,20 +307,6 @@ in
Declare it in swarm-otel-service.nix.
'';
}
{
assertion =
!(lib.elem forwarderTelemetryPort [
hyperhiveCfg.otel.telemetryPort
hyperhiveCfg.deploy.bao.otel.telemetryPort
hyperhiveCfg.deploy.nats.otel.telemetryPort
]);
message = ''
The log store's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
collides with another collector's in the host netns: the hive
collector's, the store forwarder's or the
queue forwarder's.
'';
}
];
# nspawn fails to start the container when a credential source is
@ -375,9 +358,9 @@ in
services.hyperhive.gateway.enable = lib.mkDefault true;
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
# Authenticated front door onto the loopback-only store — see the
# Authenticated front door onto the store's socket — see the
# file-top comment for why this is safe to add without touching the
# store's own (still unauthenticated, still loopback) listener at all.
# store's own (still unauthenticated, socket-only) listener at all.
# `forceSSL`: same asymmetry `swarm-ui.nix` documents —
# authelia answers a plain-http auth subrequest with 400, which
# `auth_request` cannot read as anything but a broken check, so this
@ -387,7 +370,7 @@ in
forceSSL = true;
locations = {
"/" = {
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/";
proxyPass = "${upstream}:/";
extraConfig = swarmAuthRequest;
};
@ -406,7 +389,7 @@ in
# note on `swarmAuthRequest` above), so naming it here is required
# rather than redundant.
"= /insert/opentelemetry/v1/logs" = {
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/insert/opentelemetry/v1/logs";
proxyPass = "${upstream}:/insert/opentelemetry/v1/logs";
extraConfig = ''
auth_request /__swarm_logs_ingest_authz;
'';
@ -440,7 +423,7 @@ in
"^~ /select/logsql/" = {
# No URI part, so the request path and its query string reach
# the store as the caller sent them.
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}";
proxyPass = upstream;
extraConfig = ''
auth_request /__swarm_logs_auth;
'';
@ -457,7 +440,7 @@ in
# authelia's rules for this URL admit the collector's client and
# deny every other subject (see `metricsRules` in ./swarm-authelia.nix).
"= /metrics" = {
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/metrics";
proxyPass = "${upstream}:/metrics";
extraConfig = ''
auth_request /__hive_authelia;
'';
@ -659,6 +642,12 @@ in
};
};
# The socket directory is the gateway's way in: its group is nginx's.
systemd.tmpfiles.rules = [ "d ${socketDir} 0770 root ${toString nginxGid} - -" ];
# The hive bridge carries the container's DHCP lease and its resolver.
services.hyperhive.network.enable = lib.mkDefault true;
containers.${cfg.machine} = {
autoStart = true;
ephemeral = false;
@ -668,10 +657,20 @@ in
id: source: "--load-credential=${id}:${source}"
) forwarderStoreFiles;
inherit privateNetwork;
bindMounts = caTrust.bindMount;
hostBridge = networkCfg.bridgeName;
# A bind mount, not a path both sides know: a container's `/run` is an
# nspawn tmpfs the host cannot see into. Connecting to a socket is a
# write to it.
bindMounts = {
${socketDir} = {
hostPath = socketDir;
isReadOnly = false;
};
}
// caTrust.bindMount;
config =
{ ... }:
{ pkgs, ... }:
{
imports = [
../container-modules/swarm-container.nix
@ -697,6 +696,37 @@ in
services.hyperhive.swarmContainer = { inherit privateNetwork; };
# The bridge address comes from the gateway's DHCP. `swarm-bridge-dns`
# owns resolv.conf, so dhcpcd must not write it over that unit.
networking.interfaces.eth0.useDHCP = true;
networking.dhcpcd.extraConfig = "nohook resolv.conf";
# The socket's group is the store's primary group, so that group is
# nginx's gid. A static user, not nixpkgs' DynamicUser: a dynamic
# user's group is allocated per start, and nginx is never in it.
users.groups.${socketGroup}.gid = nginxGid;
users.users.victorialogs = {
isSystemUser = true;
group = socketGroup;
};
systemd.services.victorialogs = {
serviceConfig = {
DynamicUser = lib.mkForce false;
User = "victorialogs";
Group = socketGroup;
# The store creates its socket under the process umask, so `0007`
# makes it `0770` and nginx's group can connect.
UMask = "0007";
};
# nixpkgs' readiness loop curls a TCP URL, which no `unix:` address
# answers.
postStart = lib.mkForce ''
until ${lib.getExe pkgs.curl} -s -o /dev/null --unix-socket ${socketPath} http://localhost/ping; do
sleep 1;
done
'';
};
# Authenticated as the forwarder's own OIDC client, with the
# responder's store identity: one identity for the whole container.
# It is a system credential, so the fetch unit loads it.
@ -722,12 +752,13 @@ in
enable = true;
package = deployCfg.victorialogs.package;
# ⚠️ PINNED TO LOOPBACK: upstream's default listens on every
# interface, and this endpoint accepts writes as well as reads.
# Nothing authenticates it either — upstream offers one basic-auth
# pair (`-httpAuth.username` / `-httpAuth.password`) and this
# module sets neither. The bind address is the boundary.
listenAddress = "127.0.0.1:${toString deployCfg.victorialogs.port}";
# ⚠️ A SOCKET, not upstream's default of every interface: this
# endpoint accepts writes as well as reads. Nothing authenticates it
# either — upstream offers one basic-auth pair
# (`-httpAuth.username` / `-httpAuth.password`) and this module sets
# neither. The socket's directory mode is the boundary: only the
# gateway's group can connect.
listenAddress = "unix:${socketPath}";
extraOptions = [ "-retentionPeriod=${deployCfg.victorialogs.retentionPeriod}" ];
};
@ -749,7 +780,7 @@ in
# code is not evidence.
#
# ⚠️ Like the metrics store's, that endpoint is unauthenticated — which is
# why `listenAddress` above is loopback and why nothing reaches it except
# why `listenAddress` above is a socket and why nothing reaches it except
# through the gateway, where the ingest route is authenticated. (This module
# declares a vhost.)
}

View file

@ -2262,10 +2262,10 @@ let
{
# Each old spelling lands on the new path, with its own rename warning,
# and its value reaches the unit or vhost that reads it. Ports are chosen
# so three of them collide on purpose: the bao collector's telemetry
# port equals the NATS monitor port and the forge forwarder's, and the
# NATS forwarder's equals the log store forwarder's. Each of the three
# colocation assertions must read the moved options to see it.
# so two of them collide on purpose: the bao collector's telemetry
# port equals the NATS monitor port and the forge forwarder's. Each of the
# two colocation assertions must read the moved options to see it. (The log
# store has a netns of its own, so its forwarder's port collides with none.)
name = "the per-service settings moved to deploy.* still apply from their old paths, with a rename warning each";
ok =
let
@ -2285,7 +2285,6 @@ let
nats.metricsPort = 17777;
nats.otel.telemetryPort = 8897;
otel.telemetryPort = 18889;
victorialogs.port = 19428;
forge.httpPort = 13000;
matrix.allowEncryption = true;
controller.links = [
@ -2322,7 +2321,6 @@ let
&& d.nats.metricsPort == 17777
&& d.nats.otel.telemetryPort == 8897
&& d.swarm-otel.telemetryPort == 18889
&& d.victorialogs.port == 19428
&& d.forgejo.httpPort == 13000
&& d.matrix.allowEncryption
&& d.builder.nixpkgsFlake == "path:/legacy/nixpkgs"
@ -2343,7 +2341,6 @@ let
old.containers.swarm-builder.config.systemd.services.swarm-builder.environment.SWARM_BUILDER_NIXPKGS_FLAKE
== "path:/legacy/nixpkgs"
&& (locs sw.nats.domain)."= /metrics".proxyPass == "http://127.0.0.1:17777/metrics"
&& (locs sw.victorialogs.domain)."= /metrics".proxyPass == "http://127.0.0.1:19428/metrics"
&& (locs sw.forge.domain)."/".proxyPass == "http://127.0.0.1:13000/"
&& old.containers.hive-matrix.config.services.matrix-tuwunel.settings.global.allow_encryption
&& lib.all warned [
@ -2354,7 +2351,6 @@ let
"swarm.nats.metricsPort"
"swarm.nats.otel.telemetryPort"
"swarm.otel.telemetryPort"
"swarm.victorialogs.port"
"swarm.forge.httpPort"
"swarm.matrix.allowEncryption"
"c0re.nixpkgsFlake"
@ -2365,10 +2361,8 @@ let
&& !(lib.any (w: lib.hasInfix "swarm.controller.queue" w) allLocal.warnings)
&& refused old "deploy.nats.otel.telemetryPort"
&& refused old "The forge's journal forwarder's telemetry port"
&& refused old "The log store's journal forwarder's telemetry port"
&& !(refused allLocal "deploy.nats.otel.telemetryPort")
&& !(refused allLocal "The forge's journal forwarder's telemetry port")
&& !(refused allLocal "The log store's journal forwarder's telemetry port");
&& !(refused allLocal "The forge's journal forwarder's telemetry port");
}
{
# The default render of the moved options, pinned so moving them changes
@ -2389,7 +2383,9 @@ let
&& h.deploy.nats.otel.telemetryPort == 8891
&& h.deploy.swarm-otel.telemetryPort == 8889
&& (locs h.swarm.nats.domain)."= /metrics".proxyPass == "http://127.0.0.1:7777/metrics"
&& (locs h.swarm.victorialogs.domain)."= /metrics".proxyPass == "http://127.0.0.1:9428/metrics"
&&
(locs h.swarm.victorialogs.domain)."= /metrics".proxyPass
== "http://unix:/run/swarm-victorialogs/victorialogs.sock:/metrics"
&& (locs h.swarm.forge.domain)."/".proxyPass == "http://127.0.0.1:3000/"
&& !allLocal.containers.hive-matrix.config.services.matrix-tuwunel.settings.global.allow_encryption
&& env.SWARM_CONTROLLER_MATRIX_ADMIN_SYNC_IGNORE_USERS == ""

View file

@ -511,7 +511,7 @@ let
in
h.deploy.victorialogs.enable
&& locs ? "= /metrics"
&& locs."= /metrics".proxyPass == "http://127.0.0.1:${toString h.deploy.victorialogs.port}/metrics"
&& locs."= /metrics".proxyPass == "http://unix:/run/swarm-victorialogs/victorialogs.sock:/metrics"
&& checksOf locs."= /metrics" == [ "auth_request /__hive_authelia;" ]
&& !(lib.hasInfix "error_page" locs."= /metrics".extraConfig)
&& lib.hasInfix "error_page 401 =302" locs."/".extraConfig

View file

@ -843,14 +843,14 @@ let
}
{
# Both values come from ../container-modules/swarm-container.nix.
# Four of the 16 service containers share the host netns (hive-forge, bao,
# the queue, the log store) and twelve have one of their own.
# Three of the 16 service containers share the host netns (hive-forge, bao,
# the queue) and thirteen have one of their own.
name = "a service container on the host netns runs no firewall or resolvconf of its own";
ok =
let
shared = lib.filter (c: !c.privateNetwork) (lib.attrValues serviceContainers);
in
lib.length shared == 4
lib.length shared == 3
&& lib.all (
c: !c.config.networking.firewall.enable && !c.config.networking.resolvconf.enable
) shared;
@ -990,6 +990,50 @@ let
&& bridge.serviceConfig.Group == "gateway-nginx"
&& lib.elem dir unit.serviceConfig.ReadWritePaths;
}
{
# On the bridge by DHCP, listening on a socket only the gateway's group can
# reach, so the store binds no host port. Every location that forwards to
# it goes through the socket (the authelia subrequests are the `https://`
# upstream); the read, ingest, scrape and catch-all locations are named so
# an empty filter can't pass, and the old loopback port is named nowhere.
name = "the log store runs on a netns of its own and the gateway reaches it over its socket";
ok =
let
host = serviceContainersWithCi;
net = host.services.hyperhive.network;
c = host.containers.swarm-victorialogs;
dir = "/run/swarm-victorialogs";
sock = "${dir}/victorialogs.sock";
viaSock = path: "http://unix:${sock}:${path}";
locs =
host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.victorialogs.domain}.locations;
toStore = lib.filter (
p: p != null && !(lib.hasPrefix "https://" p) && !(lib.hasPrefix "http://unix:/run/swarm-logs" p)
) (map (l: l.proxyPass) (lib.attrValues locs));
unit = c.config.systemd.services.victorialogs;
nginxGid = host.ids.gids.nginx;
in
c.privateNetwork
&& c.hostBridge == net.bridgeName
&& c.hostAddress == null
&& c.localAddress == null
&& c.config.networking.interfaces.eth0.useDHCP
&& c.config.services.victorialogs.listenAddress == "unix:${sock}"
&& locs."/".proxyPass == viaSock "/"
&& locs."= /insert/opentelemetry/v1/logs".proxyPass == viaSock "/insert/opentelemetry/v1/logs"
&& locs."^~ /select/logsql/".proxyPass == "http://unix:${sock}"
&& locs."= /metrics".proxyPass == viaSock "/metrics"
&& toStore != [ ]
&& lib.all (lib.hasPrefix "http://unix:${sock}") toStore
&& !(lib.any (lib.hasInfix "127.0.0.1") toStore)
&& c.bindMounts.${dir}.hostPath == dir
&& builtins.elem "d ${dir} 0770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules
&& c.config.users.groups.gateway-nginx.gid == nginxGid
&& unit.serviceConfig.Group == "gateway-nginx"
&& !unit.serviceConfig.DynamicUser
&& unit.serviceConfig.UMask == "0007"
&& lib.hasInfix "--unix-socket ${sock}" unit.postStart;
}
{
# On the bridge by DHCP, listening on a socket (protocol `socket` ignores the
# upstream-default `http_port`, which stays in the settings): the