vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-victoriametrics: own network namespace, gateway over a unix socket

Browse source 
The metrics store's container gets privateNetwork = true with no
interface but loopback. VictoriaMetrics listens on
unix:/run/swarm-victoriametrics/victoriametrics.sock, a host directory
(root:nginx 0770) bind-mounted into the container, and every vhost
location that forwarded to 127.0.0.1:8428 now forwards to that socket.

The store runs as a static user whose primary group is nginx's gid,
with UMask 0007, so the socket it creates is 0770 group nginx. This
relies on VictoriaMetrics >= 1.153.0 honouring the umask (the locked
nixpkgs ships 1.153.0). nixpkgs' readiness loop curls a TCP URL, so
postStart is replaced with the same loop over --unix-socket.

swarm.victoriametrics.port is removed (nothing binds it any more), with
a mkRemovedOptionModule entry, and dropped from swarm-otel's host
port-collision assertion. The container's resolver unit is dropped:
there is no interface to reach the bridge resolver through, and the
store resolves no names.

The collector scrape, the collector push and Grafana's datasource
already reach the store by its published domain, so they are unchanged.

Refs #4815
Refs #3560
This commit is contained in:
atlas 2026-10-14 14:52:31 +02:00 • committed by mara
commit 5e653ab9b5
8 changed files with 139 additions and 59 deletions
Download patch file Download diff file Expand all files Collapse all files

2
nix/module-eval/swarm-otel-identity.nix
View file

@ -282,7 +282,7 @@ let
m = locs."= /metrics" or null;
in
m != null
&& m.proxyPass == "http://127.0.0.1:${toString h.swarm.victoriametrics.port}/metrics"
&& m.proxyPass == "http://unix:/run/swarm-victoriametrics/victoriametrics.sock:/metrics"
&& lib.hasInfix "auth_request /__metrics_push_authz;" m.extraConfig
&& !(lib.hasInfix "error_page" m.extraConfig)
&& lib.hasInfix "internal;" locs."= /__metrics_push_authz".extraConfig

44
nix/module-eval/swarm-services-switch.nix
View file

@ -161,8 +161,8 @@ let
lib.filter (l: swarmServiceLinkDomains ? ${l.label}) cfg.services.hyperhive.swarm.controller.links;
# Every service container at once: the services-only host plus the CI
# runner. The CI runner and the review classifier are the two containers on
# a netns of their own.
# runner. The CI runner, the review classifier and the metrics store are the
# three containers on a netns of their own.
serviceContainersWithCi = hive {
deploy.hive-controller.enable = false;
deploy.allSwarmServices = true;
@ -484,7 +484,7 @@ let
let
shared = lib.filter (c: !c.privateNetwork) (lib.attrValues serviceContainers);
in
lib.length shared == 9
lib.length shared == 8
&& lib.all (
c: !c.config.networking.firewall.enable && !c.config.networking.resolvconf.enable
) shared;
@ -499,6 +499,44 @@ let
in
c.networking.firewall.enable && c.networking.resolvconf.enable;
}
{
# No interface but loopback, so the socket is the store's only listener
# the host can reach. Every location that forwards to the store goes
# through it; the authelia subrequest is the one `https://` upstream.
# The read, push and scrape locations are named so an empty filter
# can't pass.
name = "the metrics store runs on a netns of its own and the gateway reaches it over its socket";
ok =
let
host = serviceContainersWithCi;
c = host.containers.swarm-victoriametrics;
dir = "/run/swarm-victoriametrics";
sock = "${dir}/victoriametrics.sock";
viaSock = path: "http://unix:${sock}:${path}";
locs =
host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.victoriametrics.domain}.locations;
toStore = lib.filter (p: p != null && !(lib.hasPrefix "https://" p)) (
map (l: l.proxyPass) (lib.attrValues locs)
);
unit = c.config.systemd.services.victoriametrics;
nginxGid = host.ids.gids.nginx;
in
c.privateNetwork
&& c.hostBridge == null
&& c.hostAddress == null
&& c.config.services.victoriametrics.listenAddress == "unix:${sock}"
&& locs."= /api/v1/query".proxyPass == viaSock "/api/v1/query"
&& locs."= /opentelemetry/api/v1/push".proxyPass == viaSock "/opentelemetry/api/v1/push"
&& locs."= /metrics".proxyPass == viaSock "/metrics"
&& lib.all (lib.hasPrefix (viaSock "")) toStore
&& c.bindMounts.${dir}.hostPath == dir
&& builtins.elem "d ${dir} 0770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules
&& c.config.users.groups.gateway-nginx.gid == nginxGid
&& unit.serviceConfig.Group == "gateway-nginx"
&& !unit.serviceConfig.DynamicUser
&& unit.serviceConfig.UMask == "0007"
&& lib.hasInfix "--unix-socket ${sock}" unit.postStart;
}
{
name = "a service container that sets no stateVersion of its own is on 26.05";
ok = serviceContainers.swarm-victoriametrics.config.system.stateVersion == "26.05";