diff --git a/nix/host-modules/swarm-victoriametrics.nix b/nix/host-modules/swarm-victoriametrics.nix index 5b9b44b3..f1f0d969 100644 --- a/nix/host-modules/swarm-victoriametrics.nix +++ b/nix/host-modules/swarm-victoriametrics.nix @@ -114,6 +114,22 @@ in }; config = lib.mkIf deployCfg.victoriametrics.enable { + assertions = [ + { + assertion = lib.versionAtLeast (lib.getVersion deployCfg.victoriametrics.package) "1.153.0"; + message = '' + services.hyperhive.deploy.victoriametrics.package must be + VictoriaMetrics 1.153.0 or later (this one is + ${lib.getVersion deployCfg.victoriametrics.package}). + + The gateway reaches the store over a unix socket, and only from + 1.153.0 does VictoriaMetrics create that socket under the process + umask. An older build creates it 0600 whatever the umask, so nginx + cannot connect and every read, push and scrape fails with 502. + ''; + } + ]; + # The gateway name, inside `deployCfg.victoriametrics.enable` — that # guard is the load-bearing part. Every hive in a swarm may know this # store exists, but only the host that RUNS it may claim the name; a @@ -224,7 +240,7 @@ in }; config = - { pkgs, ... }: + { config, pkgs, ... }: { imports = [ ../container-modules/swarm-container.nix ]; @@ -262,6 +278,49 @@ in ''; }; + # Moves the store's history out of the DynamicUser layout, where + # `/var/lib/` is a symlink into `/var/lib/private/`, to the + # plain directory the static user's StateDirectory is. Runs before + # the store on every start and does nothing once + # `/var/lib/private/` is gone. When both paths hold data + # it moves neither and the store starts on the plain one. + systemd.services.victoriametrics-state-migrate = + let + old = "/var/lib/private/${config.services.victoriametrics.stateDir}"; + new = "/var/lib/${config.services.victoriametrics.stateDir}"; + in + { + description = "move VictoriaMetrics' DynamicUser state directory to its static path"; + before = [ "victoriametrics.service" ]; + requiredBy = [ "victoriametrics.service" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + SyslogIdentifier = "victoriametrics-state-migrate"; + }; + path = [ pkgs.coreutils ]; + script = '' + set -eu + old=${lib.escapeShellArg old} + new=${lib.escapeShellArg new} + if [ ! -d "$old" ] || [ -L "$old" ]; then + exit 0 + fi + if [ -L "$new" ]; then + rm "$new" + elif [ -d "$new" ]; then + if [ -n "$(ls -A "$new")" ]; then + echo "victoriametrics-state-migrate: $old and $new both hold data; moving neither" >&2 + exit 0 + fi + rmdir "$new" + fi + mv "$old" "$new" + chown -R victoriametrics:gateway-nginx "$new" + echo "victoriametrics-state-migrate: moved $old to $new" + ''; + }; + services.victoriametrics = { enable = true; package = deployCfg.victoriametrics.package; diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix index 2b772dff..c2430f99 100644 --- a/nix/module-eval/swarm-services-switch.nix +++ b/nix/module-eval/swarm-services-switch.nix @@ -169,6 +169,23 @@ let deploy.ci-runner.enable = true; }; serviceContainers = serviceContainersWithCi.containers; + + # The metrics store on a build older than its socket needs. Only its + # `assertions` are read, and the package is a stub carrying a version. + oldMetricsStore = hive { + deploy.victoriametrics.enable = true; + deploy.victoriametrics.package = pkgs.emptyDirectory // { + version = "1.152.0"; + }; + }; + # Failing assertions first, then their messages: a message is only + # well-defined when its assertion fails (nixpkgs' fileSystems one throws + # otherwise). + failedMetricsStoreVersion = + host: + lib.filter (lib.hasInfix "services.hyperhive.deploy.victoriametrics.package must be") ( + map (a: a.message) (lib.filter (a: !a.assertion) host.assertions) + ); cases = [ { # `lib.all` over an empty set holds vacuously, so the roster is counted @@ -537,6 +554,35 @@ let && unit.serviceConfig.UMask == "0007" && lib.hasInfix "--unix-socket ${sock}" unit.postStart; } + { + # The DynamicUser layout keeps the history under /var/lib/private; the + # static user's StateDirectory is the plain path. The move runs as its + # own unit ahead of the store. + name = "the metrics store's history moves from the DynamicUser path before the store starts"; + ok = + let + cfg = serviceContainers.swarm-victoriametrics.config; + m = cfg.systemd.services.victoriametrics-state-migrate; + in + cfg.services.victoriametrics.stateDir == "victoriametrics" + && builtins.elem "victoriametrics.service" m.before + && builtins.elem "victoriametrics.service" m.requiredBy + && lib.hasInfix "old=/var/lib/private/victoriametrics" m.script + && lib.hasInfix "new=/var/lib/victoriametrics" m.script + && lib.hasInfix ''mv "$old" "$new"'' m.script + && lib.hasInfix ''chown -R victoriametrics:gateway-nginx "$new"'' m.script; + } + { + # The control is the default package on the all-services host. + name = "a metrics store package older than 1.153.0 fails evaluation"; + ok = + let + old = failedMetricsStoreVersion oldMetricsStore; + in + lib.length old == 1 + && lib.hasInfix "1.152.0" (lib.head old) + && failedMetricsStoreVersion serviceContainersWithCi == [ ]; + } { name = "a service container that sets no stateVersion of its own is on 26.05"; ok = serviceContainers.swarm-victoriametrics.config.system.stateVersion == "26.05";