From 95b89970eeadd05f129406e3ac2928c699b7cdcf Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 14 Oct 2026 15:00:40 +0200 Subject: [PATCH] swarm-victoriametrics: require 1.153.0, carry the history off the DynamicUser path The socket scheme needs VictoriaMetrics 1.153.0 or later: older builds create the socket 0600 whatever the umask, so nginx could not connect and every consumer would get 502 with the module evaluating clean. deploy.victoriametrics.package is an operator option, so an assertion now refuses an older one, naming the option and the reason. Turning DynamicUser off moves the state directory from /var/lib/private/victoriametrics (behind a /var/lib/victoriametrics symlink) to a plain /var/lib/victoriametrics. A oneshot ordered before the store moves the old directory there and chowns it to the static user when the plain path is the old symlink, empty, or missing, and does nothing once the private directory is gone. Without it the persistent container's history depends on systemd's own DynamicUser-off migration. Refs #4815 --- nix/host-modules/swarm-victoriametrics.nix | 61 +++++++++++++++++++++- nix/module-eval/swarm-services-switch.nix | 46 ++++++++++++++++ 2 files changed, 106 insertions(+), 1 deletion(-) diff --git a/nix/host-modules/swarm-victoriametrics.nix b/nix/host-modules/swarm-victoriametrics.nix index 5b9b44b3..f1f0d969 100644 --- a/nix/host-modules/swarm-victoriametrics.nix +++ b/nix/host-modules/swarm-victoriametrics.nix @@ -114,6 +114,22 @@ in }; config = lib.mkIf deployCfg.victoriametrics.enable { + assertions = [ + { + assertion = lib.versionAtLeast (lib.getVersion deployCfg.victoriametrics.package) "1.153.0"; + message = '' + services.hyperhive.deploy.victoriametrics.package must be + VictoriaMetrics 1.153.0 or later (this one is + ${lib.getVersion deployCfg.victoriametrics.package}). + + The gateway reaches the store over a unix socket, and only from + 1.153.0 does VictoriaMetrics create that socket under the process + umask. An older build creates it 0600 whatever the umask, so nginx + cannot connect and every read, push and scrape fails with 502. + ''; + } + ]; + # The gateway name, inside `deployCfg.victoriametrics.enable` — that # guard is the load-bearing part. Every hive in a swarm may know this # store exists, but only the host that RUNS it may claim the name; a @@ -224,7 +240,7 @@ in }; config = - { pkgs, ... }: + { config, pkgs, ... }: { imports = [ ../container-modules/swarm-container.nix ]; @@ -262,6 +278,49 @@ in ''; }; + # Moves the store's history out of the DynamicUser layout, where + # `/var/lib/` is a symlink into `/var/lib/private/`, to the + # plain directory the static user's StateDirectory is. Runs before + # the store on every start and does nothing once + # `/var/lib/private/` is gone. When both paths hold data + # it moves neither and the store starts on the plain one. + systemd.services.victoriametrics-state-migrate = + let + old = "/var/lib/private/${config.services.victoriametrics.stateDir}"; + new = "/var/lib/${config.services.victoriametrics.stateDir}"; + in + { + description = "move VictoriaMetrics' DynamicUser state directory to its static path"; + before = [ "victoriametrics.service" ]; + requiredBy = [ "victoriametrics.service" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + SyslogIdentifier = "victoriametrics-state-migrate"; + }; + path = [ pkgs.coreutils ]; + script = '' + set -eu + old=${lib.escapeShellArg old} + new=${lib.escapeShellArg new} + if [ ! -d "$old" ] || [ -L "$old" ]; then + exit 0 + fi + if [ -L "$new" ]; then + rm "$new" + elif [ -d "$new" ]; then + if [ -n "$(ls -A "$new")" ]; then + echo "victoriametrics-state-migrate: $old and $new both hold data; moving neither" >&2 + exit 0 + fi + rmdir "$new" + fi + mv "$old" "$new" + chown -R victoriametrics:gateway-nginx "$new" + echo "victoriametrics-state-migrate: moved $old to $new" + ''; + }; + services.victoriametrics = { enable = true; package = deployCfg.victoriametrics.package; diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix index 2b772dff..c2430f99 100644 --- a/nix/module-eval/swarm-services-switch.nix +++ b/nix/module-eval/swarm-services-switch.nix @@ -169,6 +169,23 @@ let deploy.ci-runner.enable = true; }; serviceContainers = serviceContainersWithCi.containers; + + # The metrics store on a build older than its socket needs. Only its + # `assertions` are read, and the package is a stub carrying a version. + oldMetricsStore = hive { + deploy.victoriametrics.enable = true; + deploy.victoriametrics.package = pkgs.emptyDirectory // { + version = "1.152.0"; + }; + }; + # Failing assertions first, then their messages: a message is only + # well-defined when its assertion fails (nixpkgs' fileSystems one throws + # otherwise). + failedMetricsStoreVersion = + host: + lib.filter (lib.hasInfix "services.hyperhive.deploy.victoriametrics.package must be") ( + map (a: a.message) (lib.filter (a: !a.assertion) host.assertions) + ); cases = [ { # `lib.all` over an empty set holds vacuously, so the roster is counted @@ -537,6 +554,35 @@ let && unit.serviceConfig.UMask == "0007" && lib.hasInfix "--unix-socket ${sock}" unit.postStart; } + { + # The DynamicUser layout keeps the history under /var/lib/private; the + # static user's StateDirectory is the plain path. The move runs as its + # own unit ahead of the store. + name = "the metrics store's history moves from the DynamicUser path before the store starts"; + ok = + let + cfg = serviceContainers.swarm-victoriametrics.config; + m = cfg.systemd.services.victoriametrics-state-migrate; + in + cfg.services.victoriametrics.stateDir == "victoriametrics" + && builtins.elem "victoriametrics.service" m.before + && builtins.elem "victoriametrics.service" m.requiredBy + && lib.hasInfix "old=/var/lib/private/victoriametrics" m.script + && lib.hasInfix "new=/var/lib/victoriametrics" m.script + && lib.hasInfix ''mv "$old" "$new"'' m.script + && lib.hasInfix ''chown -R victoriametrics:gateway-nginx "$new"'' m.script; + } + { + # The control is the default package on the all-services host. + name = "a metrics store package older than 1.153.0 fails evaluation"; + ok = + let + old = failedMetricsStoreVersion oldMetricsStore; + in + lib.length old == 1 + && lib.hasInfix "1.152.0" (lib.head old) + && failedMetricsStoreVersion serviceContainersWithCi == [ ]; + } { name = "a service container that sets no stateVersion of its own is on 26.05"; ok = serviceContainers.swarm-victoriametrics.config.system.stateVersion == "26.05";