Watch
0
0
Fork
You've already forked hyperhive
0

swarm-victorialogs: own network namespace, gateway over a unix socket

This commit is contained in:
maia 2026-11-18 19:12:53 +01:00 • committed by mara
commit 188dbcea8b
3 changed files with 155 additions and 76 deletions

View file

@ -2262,10 +2262,10 @@ let
{
# Each old spelling lands on the new path, with its own rename warning,
# and its value reaches the unit or vhost that reads it. Ports are chosen
# so three of them collide on purpose: the bao collector's telemetry
# port equals the NATS monitor port and the forge forwarder's, and the
# NATS forwarder's equals the log store forwarder's. Each of the three
# colocation assertions must read the moved options to see it.
# so two of them collide on purpose: the bao collector's telemetry
# port equals the NATS monitor port and the forge forwarder's. Each of the
# two colocation assertions must read the moved options to see it. (The log
# store has a netns of its own, so its forwarder's port collides with none.)
name = "the per-service settings moved to deploy.* still apply from their old paths, with a rename warning each";
ok =
let
@ -2285,7 +2285,6 @@ let
nats.metricsPort = 17777;
nats.otel.telemetryPort = 8897;
otel.telemetryPort = 18889;
victorialogs.port = 19428;
forge.httpPort = 13000;
matrix.allowEncryption = true;
controller.links = [
@ -2322,7 +2321,6 @@ let
&& d.nats.metricsPort == 17777
&& d.nats.otel.telemetryPort == 8897
&& d.swarm-otel.telemetryPort == 18889
&& d.victorialogs.port == 19428
&& d.forgejo.httpPort == 13000
&& d.matrix.allowEncryption
&& d.builder.nixpkgsFlake == "path:/legacy/nixpkgs"
@ -2343,7 +2341,6 @@ let
old.containers.swarm-builder.config.systemd.services.swarm-builder.environment.SWARM_BUILDER_NIXPKGS_FLAKE
== "path:/legacy/nixpkgs"
&& (locs sw.nats.domain)."= /metrics".proxyPass == "http://127.0.0.1:17777/metrics"
&& (locs sw.victorialogs.domain)."= /metrics".proxyPass == "http://127.0.0.1:19428/metrics"
&& (locs sw.forge.domain)."/".proxyPass == "http://127.0.0.1:13000/"
&& old.containers.hive-matrix.config.services.matrix-tuwunel.settings.global.allow_encryption
&& lib.all warned [
@ -2354,7 +2351,6 @@ let
"swarm.nats.metricsPort"
"swarm.nats.otel.telemetryPort"
"swarm.otel.telemetryPort"
"swarm.victorialogs.port"
"swarm.forge.httpPort"
"swarm.matrix.allowEncryption"
"c0re.nixpkgsFlake"
@ -2365,10 +2361,8 @@ let
&& !(lib.any (w: lib.hasInfix "swarm.controller.queue" w) allLocal.warnings)
&& refused old "deploy.nats.otel.telemetryPort"
&& refused old "The forge's journal forwarder's telemetry port"
&& refused old "The log store's journal forwarder's telemetry port"
&& !(refused allLocal "deploy.nats.otel.telemetryPort")
&& !(refused allLocal "The forge's journal forwarder's telemetry port")
&& !(refused allLocal "The log store's journal forwarder's telemetry port");
&& !(refused allLocal "The forge's journal forwarder's telemetry port");
}
{
# The default render of the moved options, pinned so moving them changes
@ -2389,7 +2383,9 @@ let
&& h.deploy.nats.otel.telemetryPort == 8891
&& h.deploy.swarm-otel.telemetryPort == 8889
&& (locs h.swarm.nats.domain)."= /metrics".proxyPass == "http://127.0.0.1:7777/metrics"
&& (locs h.swarm.victorialogs.domain)."= /metrics".proxyPass == "http://127.0.0.1:9428/metrics"
&&
(locs h.swarm.victorialogs.domain)."= /metrics".proxyPass
== "http://unix:/run/swarm-victorialogs/victorialogs.sock:/metrics"
&& (locs h.swarm.forge.domain)."/".proxyPass == "http://127.0.0.1:3000/"
&& !allLocal.containers.hive-matrix.config.services.matrix-tuwunel.settings.global.allow_encryption
&& env.SWARM_CONTROLLER_MATRIX_ADMIN_SYNC_IGNORE_USERS == ""

View file

@ -511,7 +511,7 @@ let
in
h.deploy.victorialogs.enable
&& locs ? "= /metrics"
&& locs."= /metrics".proxyPass == "http://127.0.0.1:${toString h.deploy.victorialogs.port}/metrics"
&& locs."= /metrics".proxyPass == "http://unix:/run/swarm-victorialogs/victorialogs.sock:/metrics"
&& checksOf locs."= /metrics" == [ "auth_request /__hive_authelia;" ]
&& !(lib.hasInfix "error_page" locs."= /metrics".extraConfig)
&& lib.hasInfix "error_page 401 =302" locs."/".extraConfig

View file

@ -843,14 +843,14 @@ let
}
{
# Both values come from ../container-modules/swarm-container.nix.
# Four of the 16 service containers share the host netns (hive-forge, bao,
# the queue, the log store) and twelve have one of their own.
# Three of the 16 service containers share the host netns (hive-forge, bao,
# the queue) and thirteen have one of their own.
name = "a service container on the host netns runs no firewall or resolvconf of its own";
ok =
let
shared = lib.filter (c: !c.privateNetwork) (lib.attrValues serviceContainers);
in
lib.length shared == 4
lib.length shared == 3
&& lib.all (
c: !c.config.networking.firewall.enable && !c.config.networking.resolvconf.enable
) shared;
@ -990,6 +990,50 @@ let
&& bridge.serviceConfig.Group == "gateway-nginx"
&& lib.elem dir unit.serviceConfig.ReadWritePaths;
}
{
# On the bridge by DHCP, listening on a socket only the gateway's group can
# reach, so the store binds no host port. Every location that forwards to
# it goes through the socket (the authelia subrequests are the `https://`
# upstream); the read, ingest, scrape and catch-all locations are named so
# an empty filter can't pass, and the old loopback port is named nowhere.
name = "the log store runs on a netns of its own and the gateway reaches it over its socket";
ok =
let
host = serviceContainersWithCi;
net = host.services.hyperhive.network;
c = host.containers.swarm-victorialogs;
dir = "/run/swarm-victorialogs";
sock = "${dir}/victorialogs.sock";
viaSock = path: "http://unix:${sock}:${path}";
locs =
host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.victorialogs.domain}.locations;
toStore = lib.filter (
p: p != null && !(lib.hasPrefix "https://" p) && !(lib.hasPrefix "http://unix:/run/swarm-logs" p)
) (map (l: l.proxyPass) (lib.attrValues locs));
unit = c.config.systemd.services.victorialogs;
nginxGid = host.ids.gids.nginx;
in
c.privateNetwork
&& c.hostBridge == net.bridgeName
&& c.hostAddress == null
&& c.localAddress == null
&& c.config.networking.interfaces.eth0.useDHCP
&& c.config.services.victorialogs.listenAddress == "unix:${sock}"
&& locs."/".proxyPass == viaSock "/"
&& locs."= /insert/opentelemetry/v1/logs".proxyPass == viaSock "/insert/opentelemetry/v1/logs"
&& locs."^~ /select/logsql/".proxyPass == "http://unix:${sock}"
&& locs."= /metrics".proxyPass == viaSock "/metrics"
&& toStore != [ ]
&& lib.all (lib.hasPrefix "http://unix:${sock}") toStore
&& !(lib.any (lib.hasInfix "127.0.0.1") toStore)
&& c.bindMounts.${dir}.hostPath == dir
&& builtins.elem "d ${dir} 0770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules
&& c.config.users.groups.gateway-nginx.gid == nginxGid
&& unit.serviceConfig.Group == "gateway-nginx"
&& !unit.serviceConfig.DynamicUser
&& unit.serviceConfig.UMask == "0007"
&& lib.hasInfix "--unix-socket ${sock}" unit.postStart;
}
{
# On the bridge by DHCP, listening on a socket (protocol `socket` ignores the
# upstream-default `http_port`, which stays in the settings): the