vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-victoriametrics: require 1.153.0, carry the history off the DynamicUser path

Browse source 
The socket scheme needs VictoriaMetrics 1.153.0 or later: older builds
create the socket 0600 whatever the umask, so nginx could not connect
and every consumer would get 502 with the module evaluating clean.
deploy.victoriametrics.package is an operator option, so an assertion
now refuses an older one, naming the option and the reason.

Turning DynamicUser off moves the state directory from
/var/lib/private/victoriametrics (behind a /var/lib/victoriametrics
symlink) to a plain /var/lib/victoriametrics. A oneshot ordered before
the store moves the old directory there and chowns it to the static
user when the plain path is the old symlink, empty, or missing, and does
nothing once the private directory is gone. Without it the persistent
container's history depends on systemd's own DynamicUser-off migration.

Refs #4815
This commit is contained in:
atlas 2026-10-14 15:00:40 +02:00 • committed by mara
commit 95b89970ee
2 changed files with 106 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

61
nix/host-modules/swarm-victoriametrics.nix
View file

@ -114,6 +114,22 @@ in
};
config = lib.mkIf deployCfg.victoriametrics.enable {
assertions = [
{
assertion = lib.versionAtLeast (lib.getVersion deployCfg.victoriametrics.package) "1.153.0";
message = ''
services.hyperhive.deploy.victoriametrics.package must be
VictoriaMetrics 1.153.0 or later (this one is
${lib.getVersion deployCfg.victoriametrics.package}).
The gateway reaches the store over a unix socket, and only from
1.153.0 does VictoriaMetrics create that socket under the process
umask. An older build creates it 0600 whatever the umask, so nginx
cannot connect and every read, push and scrape fails with 502.
'';
}
];
# The gateway name, inside `deployCfg.victoriametrics.enable` — that
# guard is the load-bearing part. Every hive in a swarm may know this
# store exists, but only the host that RUNS it may claim the name; a
@ -224,7 +240,7 @@ in
};
config =
{ pkgs, ... }:
{ config, pkgs, ... }:
{
imports = [ ../container-modules/swarm-container.nix ];
@ -262,6 +278,49 @@ in
'';
};
# Moves the store's history out of the DynamicUser layout, where
# `/var/lib/<stateDir>` is a symlink into `/var/lib/private/`, to the
# plain directory the static user's StateDirectory is. Runs before
# the store on every start and does nothing once
# `/var/lib/private/<stateDir>` is gone. When both paths hold data
# it moves neither and the store starts on the plain one.
systemd.services.victoriametrics-state-migrate =
let
old = "/var/lib/private/${config.services.victoriametrics.stateDir}";
new = "/var/lib/${config.services.victoriametrics.stateDir}";
in
{
description = "move VictoriaMetrics' DynamicUser state directory to its static path";
before = [ "victoriametrics.service" ];
requiredBy = [ "victoriametrics.service" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
SyslogIdentifier = "victoriametrics-state-migrate";
};
path = [ pkgs.coreutils ];
script = ''
set -eu
old=${lib.escapeShellArg old}
new=${lib.escapeShellArg new}
if [ ! -d "$old" ] || [ -L "$old" ]; then
exit 0
fi
if [ -L "$new" ]; then
rm "$new"
elif [ -d "$new" ]; then
if [ -n "$(ls -A "$new")" ]; then
echo "victoriametrics-state-migrate: $old and $new both hold data; moving neither" >&2
exit 0
fi
rmdir "$new"
fi
mv "$old" "$new"
chown -R victoriametrics:gateway-nginx "$new"
echo "victoriametrics-state-migrate: moved $old to $new"
'';
};
services.victoriametrics = {
enable = true;
package = deployCfg.victoriametrics.package;

46
nix/module-eval/swarm-services-switch.nix
View file

@ -169,6 +169,23 @@ let
deploy.ci-runner.enable = true;
};
serviceContainers = serviceContainersWithCi.containers;
# The metrics store on a build older than its socket needs. Only its
# `assertions` are read, and the package is a stub carrying a version.
oldMetricsStore = hive {
deploy.victoriametrics.enable = true;
deploy.victoriametrics.package = pkgs.emptyDirectory // {
version = "1.152.0";
};
};
# Failing assertions first, then their messages: a message is only
# well-defined when its assertion fails (nixpkgs' fileSystems one throws
# otherwise).
failedMetricsStoreVersion =
host:
lib.filter (lib.hasInfix "services.hyperhive.deploy.victoriametrics.package must be") (
map (a: a.message) (lib.filter (a: !a.assertion) host.assertions)
);
cases = [
{
# `lib.all` over an empty set holds vacuously, so the roster is counted
@ -537,6 +554,35 @@ let
&& unit.serviceConfig.UMask == "0007"
&& lib.hasInfix "--unix-socket ${sock}" unit.postStart;
}
{
# The DynamicUser layout keeps the history under /var/lib/private; the
# static user's StateDirectory is the plain path. The move runs as its
# own unit ahead of the store.
name = "the metrics store's history moves from the DynamicUser path before the store starts";
ok =
let
cfg = serviceContainers.swarm-victoriametrics.config;
m = cfg.systemd.services.victoriametrics-state-migrate;
in
cfg.services.victoriametrics.stateDir == "victoriametrics"
&& builtins.elem "victoriametrics.service" m.before
&& builtins.elem "victoriametrics.service" m.requiredBy
&& lib.hasInfix "old=/var/lib/private/victoriametrics" m.script
&& lib.hasInfix "new=/var/lib/victoriametrics" m.script
&& lib.hasInfix ''mv "$old" "$new"'' m.script
&& lib.hasInfix ''chown -R victoriametrics:gateway-nginx "$new"'' m.script;
}
{
# The control is the default package on the all-services host.
name = "a metrics store package older than 1.153.0 fails evaluation";
ok =
let
old = failedMetricsStoreVersion oldMetricsStore;
in
lib.length old == 1
&& lib.hasInfix "1.152.0" (lib.head old)
&& failedMetricsStoreVersion serviceContainersWithCi == [ ];
}
{
name = "a service container that sets no stateVersion of its own is on 26.05";
ok = serviceContainers.swarm-victoriametrics.config.system.stateVersion == "26.05";