vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-victoriametrics: own network namespace, gateway over a unix socket

Browse source 
The metrics store's container gets privateNetwork = true with no
interface but loopback. VictoriaMetrics listens on
unix:/run/swarm-victoriametrics/victoriametrics.sock, a host directory
(root:nginx 0770) bind-mounted into the container, and every vhost
location that forwarded to 127.0.0.1:8428 now forwards to that socket.

The store runs as a static user whose primary group is nginx's gid,
with UMask 0007, so the socket it creates is 0770 group nginx. This
relies on VictoriaMetrics >= 1.153.0 honouring the umask (the locked
nixpkgs ships 1.153.0). nixpkgs' readiness loop curls a TCP URL, so
postStart is replaced with the same loop over --unix-socket.

swarm.victoriametrics.port is removed (nothing binds it any more), with
a mkRemovedOptionModule entry, and dropped from swarm-otel's host
port-collision assertion. The container's resolver unit is dropped:
there is no interface to reach the bridge resolver through, and the
store resolves no names.

The collector scrape, the collector push and Grafana's datasource
already reach the store by its published domain, so they are unchanged.

Refs #4815
Refs #3560
This commit is contained in:
atlas 2026-10-14 14:52:31 +02:00 • committed by mara
commit 5e653ab9b5
8 changed files with 139 additions and 59 deletions
Download patch file Download diff file Expand all files Collapse all files

5
nix/host-modules/deploy.nix
View file

@ -80,6 +80,11 @@ in
[ "services" "hyperhive" "swarm" "victoriametrics" "enable" ]
[ "services" "hyperhive" "deploy" "victoriametrics" "enable" ]
)
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "victoriametrics" "port" ] ''
VictoriaMetrics listens on a unix socket in a network namespace of its
own, and the gateway reaches it there, so it binds no host port. Remove
this definition.
'')
(lib.mkRenamedOptionModule
[ "services" "hyperhive" "swarm" "victorialogs" "enable" ]
[ "services" "hyperhive" "deploy" "victorialogs" "enable" ]

3
nix/host-modules/swarm-otel.nix
View file

@ -757,8 +757,7 @@ in
cfg.producerPort
cfg.storeProducerPort
otelCfg.collector.port
]
++ lib.optional deployCfg.victoriametrics.enable vmCfg.port;
];
all = derived ++ others;
in
lib.length (lib.unique all) == lib.length all;

3
nix/host-modules/swarm-victorialogs.nix
View file

@ -455,8 +455,7 @@ in
enable = true;
package = deployCfg.victorialogs.package;
# ⚠️ PINNED TO LOOPBACK for the same reason the metrics store is,
# and it matters more here: upstream's default listens on every
# ⚠️ PINNED TO LOOPBACK: upstream's default listens on every
# interface, and this endpoint accepts writes as well as reads.
# Nothing authenticates it either — upstream offers one basic-auth
# pair (`-httpAuth.username` / `-httpAuth.password`) and this

21
nix/host-modules/swarm-victoriametrics-service.nix
View file

@ -1,5 +1,5 @@
# The swarm's metrics store as every hive sees it: the name it answers on and
# its port, identical on every host. What the host running it decides, and the
# The swarm's metrics store as every hive sees it: the name it answers on,
# identical on every host. What the host running it decides, and the
# container itself, are in ./swarm-victoriametrics.nix.
{
lib,
@ -16,8 +16,8 @@ let
domainBase = if swarmDomain == null then "invalid" else swarmDomain;
in
{
# What the store IS from any hive's point of view: the name it answers on and
# the port. `enable`, `package` and `retentionPeriod` are decisions of the
# What the store IS from any hive's point of view: the name it answers on.
# `enable`, `package` and `retentionPeriod` are decisions of the
# host that runs it and live under `deploy.*`.
options.services.hyperhive.swarm.victoriametrics = {
machine = lib.mkOption {
@ -42,18 +42,5 @@ in
child decides which CA may sign it.
'';
};
port = lib.mkOption {
type = lib.types.port;
default = 8428;
description = ''
Port VictoriaMetrics listens on, bound to loopback only on the
host that runs it: its write endpoint is unauthenticated, so the
gateway's vhost there is the only route to it. Upstream's own
default, kept so an operator reading VictoriaMetrics
documentation finds what they expect.
'';
};
};
}

113
nix/host-modules/swarm-victoriametrics.nix
View file

@ -18,13 +18,23 @@
let
cfg = config.services.hyperhive.swarm.victoriametrics;
deployCfg = config.services.hyperhive.deploy;
networkCfg = config.services.hyperhive.network;
hyperhiveCfg = config.services.hyperhive;
gatewayCfg = hyperhiveCfg.gateway;
# Shared host netns, like every sibling swarm container: the gateway
# reaches this at 127.0.0.1:<port>.
privateNetwork = false;
# A netns of its own with no interface but loopback: the store makes no
# outbound calls, and the gateway reaches it over `socketPath` below, so
# nothing on the host or the hive bridge can dial its listener.
privateNetwork = true;
# Fixed, not an option: the directory's mode is what keeps the socket
# private, so it holds this one socket and nothing else.
socketDir = "/run/swarm-victoriametrics";
socketPath = "${socketDir}/victoriametrics.sock";
upstream = path: "http://unix:${socketPath}:${path}";
# The gid the gateway's nginx runs as. Only the number crosses into the
# container, which knows it as `gateway-nginx`.
nginxGid = config.ids.gids.nginx;
# The store's read API: the only part of it the vhost serves without a
# credential. Each location's `proxy_pass` carries the path, so the store
@ -64,7 +74,7 @@ let
];
readLocations =
let
to = path: { proxyPass = "http://127.0.0.1:${toString cfg.port}${path}"; };
to = path: { proxyPass = upstream path; };
in
lib.listToAttrs (
map (p: lib.nameValuePair "= ${p}" (to p)) readPaths
@ -72,8 +82,8 @@ let
);
in
{
# What the store IS from any hive's point of view, the name it answers on and
# the port, is `swarm.victoriametrics` in ./swarm-victoriametrics-service.nix.
# What the store IS from any hive's point of view, the name it answers on,
# is `swarm.victoriametrics` in ./swarm-victoriametrics-service.nix.
# `enable`, `package` and `retentionPeriod` are decisions of the host that
# runs it and live under `deploy.*`.
@ -111,8 +121,7 @@ in
# have.
services.hyperhive.gateway.localNames = [ cfg.domain ];
# This host serves the vhost, and the container behind it resolves
# through the hive's dnsmasq.
# This host serves the vhost, and the hive's dnsmasq answers its name.
services.hyperhive.gateway.enable = lib.mkDefault true;
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
@ -138,7 +147,7 @@ in
# follow it and POST its batch at a login page that answers 200 —
# ingest reporting healthy while storing nothing.
"= /opentelemetry/api/v1/push" = {
proxyPass = "http://127.0.0.1:${toString cfg.port}/opentelemetry/api/v1/push";
proxyPass = upstream "/opentelemetry/api/v1/push";
extraConfig = ''
auth_request /__metrics_push_authz;
'';
@ -153,7 +162,7 @@ in
# `publishedScrapeTargets.victoriametrics` in ./swarm-otel-service.nix;
# the two must agree, since that string is also the token's audience.
"= /metrics" = {
proxyPass = "http://127.0.0.1:${toString cfg.port}/metrics";
proxyPass = upstream "/metrics";
extraConfig = ''
auth_request /__metrics_push_authz;
'';
@ -188,6 +197,17 @@ in
};
};
# The socket directory, created host-side before the container starts:
# nixos-container refuses to start when a bind source is missing.
#
# Root-owned and group nginx, `0770`: the store creates its socket here
# through the group (its uid is allocated inside the container, so the
# host cannot name it), and the gateway traverses it the same way.
# Everything else is kept out.
systemd.tmpfiles.rules = [
"d ${socketDir} 0770 root ${toString nginxGid} - -"
];
containers.${cfg.machine} = {
autoStart = true;
ephemeral = false;
@ -196,31 +216,62 @@ in
extraFlags = [ "--link-journal=host" ];
inherit privateNetwork;
config =
{ ... }:
{
imports = [
../container-modules/swarm-container.nix
(import ../container-modules/swarm-container-resolver.nix {
inherit (networkCfg) bridgeIp;
dnsConsumers = [ "victoriametrics.service" ];
})
];
# A bind mount rather than a path both sides happen to know: a
# container's `/run` is an nspawn tmpfs the host cannot see into.
bindMounts.${socketDir} = {
hostPath = socketDir;
isReadOnly = false;
};
services.hyperhive.swarmContainer = { inherit privateNetwork; };
config =
{ pkgs, ... }:
{
imports = [ ../container-modules/swarm-container.nix ];
# No resolver unit: with no interface but loopback there is no
# nameserver to reach, and the store resolves no names.
services.hyperhive.swarmContainer = {
inherit privateNetwork;
writesOwnResolvConf = false;
};
# The socket's group is the store's primary group, so that group is
# nginx's gid. A static user, not nixpkgs' DynamicUser: a dynamic
# user's group is allocated per start, and nginx is never in it.
users.groups.gateway-nginx.gid = nginxGid;
users.users.victoriametrics = {
isSystemUser = true;
group = "gateway-nginx";
};
systemd.services.victoriametrics = {
serviceConfig = {
DynamicUser = lib.mkForce false;
User = "victoriametrics";
Group = "gateway-nginx";
# The store creates its socket under the process umask, so
# `0007` makes it `0770` and nginx's group can connect.
UMask = "0007";
};
# nixpkgs' readiness loop curls `http://<listenAddress>/ping`,
# which no `unix:` address answers.
postStart = lib.mkForce ''
until ${lib.getExe pkgs.curl} -s -o /dev/null --unix-socket ${socketPath} http://localhost/ping; do
sleep 1;
done
'';
};
services.victoriametrics = {
enable = true;
package = deployCfg.victoriametrics.package;
retentionPeriod = deployCfg.victoriametrics.retentionPeriod;
# ⚠️ PINNED TO LOOPBACK, and this is a correction rather than a
# preference: upstream's default is `:8428`, i.e. every
# interface. The gateway is the only intended client and it is on
# this host, so binding wider would publish an unauthenticated
# write endpoint (see the OTLP note below) to whatever the host
# is reachable on.
listenAddress = "127.0.0.1:${toString cfg.port}";
# ⚠️ A socket, not upstream's default `:8428` on every interface:
# the gateway is the only intended client, and this listener
# carries an unauthenticated write endpoint (see the OTLP note
# below).
listenAddress = "unix:${socketPath}";
};
};
};
@ -235,8 +286,8 @@ in
# deployment ever needs them.
#
# ⚠️ That endpoint is unauthenticated, which is why `listenAddress` above is
# loopback: the only route to it from off-host is the vhost's ingest
# location, which authenticates. The collector is still the only writer, but
# a socket only the gateway can reach: the only route to it is the vhost's
# ingest location, which authenticates. The collector is still the only writer, but
# it now arrives by the swarm name rather than over loopback, because it need
# not share a host with this store.
}