diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 09c8f046..8273a55c 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -168,10 +168,8 @@ Grafana start — see [`secrets.md`](secrets.md). **Where the data comes from.** The swarm's OTEL collector, below. -Neither container is reachable except through the gateway: each listens -on a unix socket only the gateway's nginx can open, and VictoriaMetrics -runs in a network namespace of its own with no interface but loopback. -VictoriaMetrics authenticates nothing itself. The +Neither container is reachable except through the gateway: both bind +loopback, and VictoriaMetrics authenticates nothing itself. The `metrics.` vhost forwards only the store's read API (the Prometheus query, series, label and metadata routes, plus the vmui web UI) without a credential. It forwards the collector's OTLP push, and the @@ -208,8 +206,9 @@ swarm. | `swarm.victorialogs.domain` | Only to rename it. | | `swarm.victorialogs.port` | Only if something else on the services host already claims `9428`. | -It binds loopback and takes no credential of its own: the gateway vhost is -the only way in, and the collector is the only intended writer. +Like the metrics store, it binds loopback and takes no credential of its +own: the gateway vhost is the only way in, and the collector is the only +intended writer. The collector also scrapes this store's own metrics, at `https:///metrics`, so a collector on a host diff --git a/nix/host-modules/deploy.nix b/nix/host-modules/deploy.nix index c555e92b..4f905616 100644 --- a/nix/host-modules/deploy.nix +++ b/nix/host-modules/deploy.nix @@ -80,11 +80,6 @@ in [ "services" "hyperhive" "swarm" "victoriametrics" "enable" ] [ "services" "hyperhive" "deploy" "victoriametrics" "enable" ] ) - (lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "victoriametrics" "port" ] '' - VictoriaMetrics listens on a unix socket in a network namespace of its - own, and the gateway reaches it there, so it binds no host port. Remove - this definition. - '') (lib.mkRenamedOptionModule [ "services" "hyperhive" "swarm" "victorialogs" "enable" ] [ "services" "hyperhive" "deploy" "victorialogs" "enable" ] diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index f638b52d..6867e713 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -757,7 +757,8 @@ in cfg.producerPort cfg.storeProducerPort otelCfg.collector.port - ]; + ] + ++ lib.optional deployCfg.victoriametrics.enable vmCfg.port; all = derived ++ others; in lib.length (lib.unique all) == lib.length all; diff --git a/nix/host-modules/swarm-victorialogs.nix b/nix/host-modules/swarm-victorialogs.nix index 0d9a24f2..15ecac06 100644 --- a/nix/host-modules/swarm-victorialogs.nix +++ b/nix/host-modules/swarm-victorialogs.nix @@ -455,7 +455,8 @@ in enable = true; package = deployCfg.victorialogs.package; - # ⚠️ PINNED TO LOOPBACK: upstream's default listens on every + # ⚠️ PINNED TO LOOPBACK for the same reason the metrics store is, + # and it matters more here: upstream's default listens on every # interface, and this endpoint accepts writes as well as reads. # Nothing authenticates it either — upstream offers one basic-auth # pair (`-httpAuth.username` / `-httpAuth.password`) and this diff --git a/nix/host-modules/swarm-victoriametrics-service.nix b/nix/host-modules/swarm-victoriametrics-service.nix index 83a3ad7f..61a61649 100644 --- a/nix/host-modules/swarm-victoriametrics-service.nix +++ b/nix/host-modules/swarm-victoriametrics-service.nix @@ -1,5 +1,5 @@ -# The swarm's metrics store as every hive sees it: the name it answers on, -# identical on every host. What the host running it decides, and the +# The swarm's metrics store as every hive sees it: the name it answers on and +# its port, identical on every host. What the host running it decides, and the # container itself, are in ./swarm-victoriametrics.nix. { lib, @@ -16,8 +16,8 @@ let domainBase = if swarmDomain == null then "invalid" else swarmDomain; in { - # What the store IS from any hive's point of view: the name it answers on. - # `enable`, `package` and `retentionPeriod` are decisions of the + # What the store IS from any hive's point of view: the name it answers on and + # the port. `enable`, `package` and `retentionPeriod` are decisions of the # host that runs it and live under `deploy.*`. options.services.hyperhive.swarm.victoriametrics = { machine = lib.mkOption { @@ -42,5 +42,18 @@ in child decides which CA may sign it. ''; }; + + port = lib.mkOption { + type = lib.types.port; + default = 8428; + description = '' + Port VictoriaMetrics listens on, bound to loopback only on the + host that runs it: its write endpoint is unauthenticated, so the + gateway's vhost there is the only route to it. Upstream's own + default, kept so an operator reading VictoriaMetrics + documentation finds what they expect. + ''; + }; + }; } diff --git a/nix/host-modules/swarm-victoriametrics.nix b/nix/host-modules/swarm-victoriametrics.nix index f1f0d969..ffd6e828 100644 --- a/nix/host-modules/swarm-victoriametrics.nix +++ b/nix/host-modules/swarm-victoriametrics.nix @@ -18,23 +18,13 @@ let cfg = config.services.hyperhive.swarm.victoriametrics; deployCfg = config.services.hyperhive.deploy; + networkCfg = config.services.hyperhive.network; hyperhiveCfg = config.services.hyperhive; gatewayCfg = hyperhiveCfg.gateway; - # A netns of its own with no interface but loopback: the store makes no - # outbound calls, and the gateway reaches it over `socketPath` below, so - # nothing on the host or the hive bridge can dial its listener. - privateNetwork = true; - - # Fixed, not an option: the directory's mode is what keeps the socket - # private, so it holds this one socket and nothing else. - socketDir = "/run/swarm-victoriametrics"; - socketPath = "${socketDir}/victoriametrics.sock"; - upstream = path: "http://unix:${socketPath}:${path}"; - - # The gid the gateway's nginx runs as. Only the number crosses into the - # container, which knows it as `gateway-nginx`. - nginxGid = config.ids.gids.nginx; + # Shared host netns, like every sibling swarm container: the gateway + # reaches this at 127.0.0.1:. + privateNetwork = false; # The store's read API: the only part of it the vhost serves without a # credential. Each location's `proxy_pass` carries the path, so the store @@ -74,7 +64,7 @@ let ]; readLocations = let - to = path: { proxyPass = upstream path; }; + to = path: { proxyPass = "http://127.0.0.1:${toString cfg.port}${path}"; }; in lib.listToAttrs ( map (p: lib.nameValuePair "= ${p}" (to p)) readPaths @@ -82,8 +72,8 @@ let ); in { - # What the store IS from any hive's point of view, the name it answers on, - # is `swarm.victoriametrics` in ./swarm-victoriametrics-service.nix. + # What the store IS from any hive's point of view, the name it answers on and + # the port, is `swarm.victoriametrics` in ./swarm-victoriametrics-service.nix. # `enable`, `package` and `retentionPeriod` are decisions of the host that # runs it and live under `deploy.*`. @@ -114,22 +104,6 @@ in }; config = lib.mkIf deployCfg.victoriametrics.enable { - assertions = [ - { - assertion = lib.versionAtLeast (lib.getVersion deployCfg.victoriametrics.package) "1.153.0"; - message = '' - services.hyperhive.deploy.victoriametrics.package must be - VictoriaMetrics 1.153.0 or later (this one is - ${lib.getVersion deployCfg.victoriametrics.package}). - - The gateway reaches the store over a unix socket, and only from - 1.153.0 does VictoriaMetrics create that socket under the process - umask. An older build creates it 0600 whatever the umask, so nginx - cannot connect and every read, push and scrape fails with 502. - ''; - } - ]; - # The gateway name, inside `deployCfg.victoriametrics.enable` — that # guard is the load-bearing part. Every hive in a swarm may know this # store exists, but only the host that RUNS it may claim the name; a @@ -137,7 +111,8 @@ in # have. services.hyperhive.gateway.localNames = [ cfg.domain ]; - # This host serves the vhost, and the hive's dnsmasq answers its name. + # This host serves the vhost, and the container behind it resolves + # through the hive's dnsmasq. services.hyperhive.gateway.enable = lib.mkDefault true; services.hyperhive.gateway.dns.enable = lib.mkDefault true; @@ -163,7 +138,7 @@ in # follow it and POST its batch at a login page that answers 200 — # ingest reporting healthy while storing nothing. "= /opentelemetry/api/v1/push" = { - proxyPass = upstream "/opentelemetry/api/v1/push"; + proxyPass = "http://127.0.0.1:${toString cfg.port}/opentelemetry/api/v1/push"; extraConfig = '' auth_request /__metrics_push_authz; ''; @@ -178,7 +153,7 @@ in # `publishedScrapeTargets.victoriametrics` in ./swarm-otel-service.nix; # the two must agree, since that string is also the token's audience. "= /metrics" = { - proxyPass = upstream "/metrics"; + proxyPass = "http://127.0.0.1:${toString cfg.port}/metrics"; extraConfig = '' auth_request /__metrics_push_authz; ''; @@ -213,17 +188,6 @@ in }; }; - # The socket directory, created host-side before the container starts: - # nixos-container refuses to start when a bind source is missing. - # - # Root-owned and group nginx, `0770`: the store creates its socket here - # through the group (its uid is allocated inside the container, so the - # host cannot name it), and the gateway traverses it the same way. - # Everything else is kept out. - systemd.tmpfiles.rules = [ - "d ${socketDir} 0770 root ${toString nginxGid} - -" - ]; - containers.${cfg.machine} = { autoStart = true; ephemeral = false; @@ -232,105 +196,31 @@ in extraFlags = [ "--link-journal=host" ]; inherit privateNetwork; - # A bind mount rather than a path both sides happen to know: a - # container's `/run` is an nspawn tmpfs the host cannot see into. - bindMounts.${socketDir} = { - hostPath = socketDir; - isReadOnly = false; - }; - config = - { config, pkgs, ... }: + { ... }: { - imports = [ ../container-modules/swarm-container.nix ]; + imports = [ + ../container-modules/swarm-container.nix + (import ../container-modules/swarm-container-resolver.nix { + inherit (networkCfg) bridgeIp; + dnsConsumers = [ "victoriametrics.service" ]; + }) + ]; - # No resolver unit: with no interface but loopback there is no - # nameserver to reach, and the store resolves no names. - services.hyperhive.swarmContainer = { - inherit privateNetwork; - writesOwnResolvConf = false; - }; - - # The socket's group is the store's primary group, so that group is - # nginx's gid. A static user, not nixpkgs' DynamicUser: a dynamic - # user's group is allocated per start, and nginx is never in it. - users.groups.gateway-nginx.gid = nginxGid; - users.users.victoriametrics = { - isSystemUser = true; - group = "gateway-nginx"; - }; - - systemd.services.victoriametrics = { - serviceConfig = { - DynamicUser = lib.mkForce false; - User = "victoriametrics"; - Group = "gateway-nginx"; - # The store creates its socket under the process umask, so - # `0007` makes it `0770` and nginx's group can connect. - UMask = "0007"; - }; - # nixpkgs' readiness loop curls `http:///ping`, - # which no `unix:` address answers. - postStart = lib.mkForce '' - until ${lib.getExe pkgs.curl} -s -o /dev/null --unix-socket ${socketPath} http://localhost/ping; do - sleep 1; - done - ''; - }; - - # Moves the store's history out of the DynamicUser layout, where - # `/var/lib/` is a symlink into `/var/lib/private/`, to the - # plain directory the static user's StateDirectory is. Runs before - # the store on every start and does nothing once - # `/var/lib/private/` is gone. When both paths hold data - # it moves neither and the store starts on the plain one. - systemd.services.victoriametrics-state-migrate = - let - old = "/var/lib/private/${config.services.victoriametrics.stateDir}"; - new = "/var/lib/${config.services.victoriametrics.stateDir}"; - in - { - description = "move VictoriaMetrics' DynamicUser state directory to its static path"; - before = [ "victoriametrics.service" ]; - requiredBy = [ "victoriametrics.service" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - SyslogIdentifier = "victoriametrics-state-migrate"; - }; - path = [ pkgs.coreutils ]; - script = '' - set -eu - old=${lib.escapeShellArg old} - new=${lib.escapeShellArg new} - if [ ! -d "$old" ] || [ -L "$old" ]; then - exit 0 - fi - if [ -L "$new" ]; then - rm "$new" - elif [ -d "$new" ]; then - if [ -n "$(ls -A "$new")" ]; then - echo "victoriametrics-state-migrate: $old and $new both hold data; moving neither" >&2 - exit 0 - fi - rmdir "$new" - fi - mv "$old" "$new" - chown -R victoriametrics:gateway-nginx "$new" - echo "victoriametrics-state-migrate: moved $old to $new" - ''; - }; + services.hyperhive.swarmContainer = { inherit privateNetwork; }; services.victoriametrics = { enable = true; package = deployCfg.victoriametrics.package; retentionPeriod = deployCfg.victoriametrics.retentionPeriod; - # ⚠️ A socket, not upstream's default `:8428` on every interface: - # the gateway is the only intended client, and this listener - # carries an unauthenticated write endpoint (see the OTLP note - # below). - listenAddress = "unix:${socketPath}"; + # ⚠️ PINNED TO LOOPBACK, and this is a correction rather than a + # preference: upstream's default is `:8428`, i.e. every + # interface. The gateway is the only intended client and it is on + # this host, so binding wider would publish an unauthenticated + # write endpoint (see the OTLP note below) to whatever the host + # is reachable on. + listenAddress = "127.0.0.1:${toString cfg.port}"; }; }; }; @@ -345,8 +235,8 @@ in # deployment ever needs them. # # ⚠️ That endpoint is unauthenticated, which is why `listenAddress` above is - # a socket only the gateway can reach: the only route to it is the vhost's - # ingest location, which authenticates. The collector is still the only writer, but + # loopback: the only route to it from off-host is the vhost's ingest + # location, which authenticates. The collector is still the only writer, but # it now arrives by the swarm name rather than over loopback, because it need # not share a host with this store. } diff --git a/nix/module-eval/swarm-otel-identity.nix b/nix/module-eval/swarm-otel-identity.nix index 558ac020..0172ff85 100644 --- a/nix/module-eval/swarm-otel-identity.nix +++ b/nix/module-eval/swarm-otel-identity.nix @@ -282,7 +282,7 @@ let m = locs."= /metrics" or null; in m != null - && m.proxyPass == "http://unix:/run/swarm-victoriametrics/victoriametrics.sock:/metrics" + && m.proxyPass == "http://127.0.0.1:${toString h.swarm.victoriametrics.port}/metrics" && lib.hasInfix "auth_request /__metrics_push_authz;" m.extraConfig && !(lib.hasInfix "error_page" m.extraConfig) && lib.hasInfix "internal;" locs."= /__metrics_push_authz".extraConfig diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix index c2430f99..dcd92397 100644 --- a/nix/module-eval/swarm-services-switch.nix +++ b/nix/module-eval/swarm-services-switch.nix @@ -161,31 +161,14 @@ let lib.filter (l: swarmServiceLinkDomains ? ${l.label}) cfg.services.hyperhive.swarm.controller.links; # Every service container at once: the services-only host plus the CI - # runner. The CI runner, the review classifier and the metrics store are the - # three containers on a netns of their own. + # runner. The CI runner and the review classifier are the two containers on + # a netns of their own. serviceContainersWithCi = hive { deploy.hive-controller.enable = false; deploy.allSwarmServices = true; deploy.ci-runner.enable = true; }; serviceContainers = serviceContainersWithCi.containers; - - # The metrics store on a build older than its socket needs. Only its - # `assertions` are read, and the package is a stub carrying a version. - oldMetricsStore = hive { - deploy.victoriametrics.enable = true; - deploy.victoriametrics.package = pkgs.emptyDirectory // { - version = "1.152.0"; - }; - }; - # Failing assertions first, then their messages: a message is only - # well-defined when its assertion fails (nixpkgs' fileSystems one throws - # otherwise). - failedMetricsStoreVersion = - host: - lib.filter (lib.hasInfix "services.hyperhive.deploy.victoriametrics.package must be") ( - map (a: a.message) (lib.filter (a: !a.assertion) host.assertions) - ); cases = [ { # `lib.all` over an empty set holds vacuously, so the roster is counted @@ -501,7 +484,7 @@ let let shared = lib.filter (c: !c.privateNetwork) (lib.attrValues serviceContainers); in - lib.length shared == 8 + lib.length shared == 9 && lib.all ( c: !c.config.networking.firewall.enable && !c.config.networking.resolvconf.enable ) shared; @@ -516,73 +499,6 @@ let in c.networking.firewall.enable && c.networking.resolvconf.enable; } - { - # No interface but loopback, so the socket is the store's only listener - # the host can reach. Every location that forwards to the store goes - # through it; the authelia subrequest is the one `https://` upstream. - # The read, push and scrape locations are named so an empty filter - # can't pass. - name = "the metrics store runs on a netns of its own and the gateway reaches it over its socket"; - ok = - let - host = serviceContainersWithCi; - c = host.containers.swarm-victoriametrics; - dir = "/run/swarm-victoriametrics"; - sock = "${dir}/victoriametrics.sock"; - viaSock = path: "http://unix:${sock}:${path}"; - locs = - host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.victoriametrics.domain}.locations; - toStore = lib.filter (p: p != null && !(lib.hasPrefix "https://" p)) ( - map (l: l.proxyPass) (lib.attrValues locs) - ); - unit = c.config.systemd.services.victoriametrics; - nginxGid = host.ids.gids.nginx; - in - c.privateNetwork - && c.hostBridge == null - && c.hostAddress == null - && c.config.services.victoriametrics.listenAddress == "unix:${sock}" - && locs."= /api/v1/query".proxyPass == viaSock "/api/v1/query" - && locs."= /opentelemetry/api/v1/push".proxyPass == viaSock "/opentelemetry/api/v1/push" - && locs."= /metrics".proxyPass == viaSock "/metrics" - && lib.all (lib.hasPrefix (viaSock "")) toStore - && c.bindMounts.${dir}.hostPath == dir - && builtins.elem "d ${dir} 0770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules - && c.config.users.groups.gateway-nginx.gid == nginxGid - && unit.serviceConfig.Group == "gateway-nginx" - && !unit.serviceConfig.DynamicUser - && unit.serviceConfig.UMask == "0007" - && lib.hasInfix "--unix-socket ${sock}" unit.postStart; - } - { - # The DynamicUser layout keeps the history under /var/lib/private; the - # static user's StateDirectory is the plain path. The move runs as its - # own unit ahead of the store. - name = "the metrics store's history moves from the DynamicUser path before the store starts"; - ok = - let - cfg = serviceContainers.swarm-victoriametrics.config; - m = cfg.systemd.services.victoriametrics-state-migrate; - in - cfg.services.victoriametrics.stateDir == "victoriametrics" - && builtins.elem "victoriametrics.service" m.before - && builtins.elem "victoriametrics.service" m.requiredBy - && lib.hasInfix "old=/var/lib/private/victoriametrics" m.script - && lib.hasInfix "new=/var/lib/victoriametrics" m.script - && lib.hasInfix ''mv "$old" "$new"'' m.script - && lib.hasInfix ''chown -R victoriametrics:gateway-nginx "$new"'' m.script; - } - { - # The control is the default package on the all-services host. - name = "a metrics store package older than 1.153.0 fails evaluation"; - ok = - let - old = failedMetricsStoreVersion oldMetricsStore; - in - lib.length old == 1 - && lib.hasInfix "1.152.0" (lib.head old) - && failedMetricsStoreVersion serviceContainersWithCi == [ ]; - } { name = "a service container that sets no stateVersion of its own is on 26.05"; ok = serviceContainers.swarm-victoriametrics.config.system.stateVersion == "26.05";