diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 8273a55c..09c8f046 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -168,8 +168,10 @@ Grafana start — see [`secrets.md`](secrets.md). **Where the data comes from.** The swarm's OTEL collector, below. -Neither container is reachable except through the gateway: both bind -loopback, and VictoriaMetrics authenticates nothing itself. The +Neither container is reachable except through the gateway: each listens +on a unix socket only the gateway's nginx can open, and VictoriaMetrics +runs in a network namespace of its own with no interface but loopback. +VictoriaMetrics authenticates nothing itself. The `metrics.` vhost forwards only the store's read API (the Prometheus query, series, label and metadata routes, plus the vmui web UI) without a credential. It forwards the collector's OTLP push, and the @@ -206,9 +208,8 @@ swarm. | `swarm.victorialogs.domain` | Only to rename it. | | `swarm.victorialogs.port` | Only if something else on the services host already claims `9428`. | -Like the metrics store, it binds loopback and takes no credential of its -own: the gateway vhost is the only way in, and the collector is the only -intended writer. +It binds loopback and takes no credential of its own: the gateway vhost is +the only way in, and the collector is the only intended writer. The collector also scrapes this store's own metrics, at `https:///metrics`, so a collector on a host diff --git a/nix/host-modules/deploy.nix b/nix/host-modules/deploy.nix index 4f905616..c555e92b 100644 --- a/nix/host-modules/deploy.nix +++ b/nix/host-modules/deploy.nix @@ -80,6 +80,11 @@ in [ "services" "hyperhive" "swarm" "victoriametrics" "enable" ] [ "services" "hyperhive" "deploy" "victoriametrics" "enable" ] ) + (lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "victoriametrics" "port" ] '' + VictoriaMetrics listens on a unix socket in a network namespace of its + own, and the gateway reaches it there, so it binds no host port. Remove + this definition. + '') (lib.mkRenamedOptionModule [ "services" "hyperhive" "swarm" "victorialogs" "enable" ] [ "services" "hyperhive" "deploy" "victorialogs" "enable" ] diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 6867e713..f638b52d 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -757,8 +757,7 @@ in cfg.producerPort cfg.storeProducerPort otelCfg.collector.port - ] - ++ lib.optional deployCfg.victoriametrics.enable vmCfg.port; + ]; all = derived ++ others; in lib.length (lib.unique all) == lib.length all; diff --git a/nix/host-modules/swarm-victorialogs.nix b/nix/host-modules/swarm-victorialogs.nix index 15ecac06..0d9a24f2 100644 --- a/nix/host-modules/swarm-victorialogs.nix +++ b/nix/host-modules/swarm-victorialogs.nix @@ -455,8 +455,7 @@ in enable = true; package = deployCfg.victorialogs.package; - # ⚠️ PINNED TO LOOPBACK for the same reason the metrics store is, - # and it matters more here: upstream's default listens on every + # ⚠️ PINNED TO LOOPBACK: upstream's default listens on every # interface, and this endpoint accepts writes as well as reads. # Nothing authenticates it either — upstream offers one basic-auth # pair (`-httpAuth.username` / `-httpAuth.password`) and this diff --git a/nix/host-modules/swarm-victoriametrics-service.nix b/nix/host-modules/swarm-victoriametrics-service.nix index 61a61649..83a3ad7f 100644 --- a/nix/host-modules/swarm-victoriametrics-service.nix +++ b/nix/host-modules/swarm-victoriametrics-service.nix @@ -1,5 +1,5 @@ -# The swarm's metrics store as every hive sees it: the name it answers on and -# its port, identical on every host. What the host running it decides, and the +# The swarm's metrics store as every hive sees it: the name it answers on, +# identical on every host. What the host running it decides, and the # container itself, are in ./swarm-victoriametrics.nix. { lib, @@ -16,8 +16,8 @@ let domainBase = if swarmDomain == null then "invalid" else swarmDomain; in { - # What the store IS from any hive's point of view: the name it answers on and - # the port. `enable`, `package` and `retentionPeriod` are decisions of the + # What the store IS from any hive's point of view: the name it answers on. + # `enable`, `package` and `retentionPeriod` are decisions of the # host that runs it and live under `deploy.*`. options.services.hyperhive.swarm.victoriametrics = { machine = lib.mkOption { @@ -42,18 +42,5 @@ in child decides which CA may sign it. ''; }; - - port = lib.mkOption { - type = lib.types.port; - default = 8428; - description = '' - Port VictoriaMetrics listens on, bound to loopback only on the - host that runs it: its write endpoint is unauthenticated, so the - gateway's vhost there is the only route to it. Upstream's own - default, kept so an operator reading VictoriaMetrics - documentation finds what they expect. - ''; - }; - }; } diff --git a/nix/host-modules/swarm-victoriametrics.nix b/nix/host-modules/swarm-victoriametrics.nix index ffd6e828..5b9b44b3 100644 --- a/nix/host-modules/swarm-victoriametrics.nix +++ b/nix/host-modules/swarm-victoriametrics.nix @@ -18,13 +18,23 @@ let cfg = config.services.hyperhive.swarm.victoriametrics; deployCfg = config.services.hyperhive.deploy; - networkCfg = config.services.hyperhive.network; hyperhiveCfg = config.services.hyperhive; gatewayCfg = hyperhiveCfg.gateway; - # Shared host netns, like every sibling swarm container: the gateway - # reaches this at 127.0.0.1:. - privateNetwork = false; + # A netns of its own with no interface but loopback: the store makes no + # outbound calls, and the gateway reaches it over `socketPath` below, so + # nothing on the host or the hive bridge can dial its listener. + privateNetwork = true; + + # Fixed, not an option: the directory's mode is what keeps the socket + # private, so it holds this one socket and nothing else. + socketDir = "/run/swarm-victoriametrics"; + socketPath = "${socketDir}/victoriametrics.sock"; + upstream = path: "http://unix:${socketPath}:${path}"; + + # The gid the gateway's nginx runs as. Only the number crosses into the + # container, which knows it as `gateway-nginx`. + nginxGid = config.ids.gids.nginx; # The store's read API: the only part of it the vhost serves without a # credential. Each location's `proxy_pass` carries the path, so the store @@ -64,7 +74,7 @@ let ]; readLocations = let - to = path: { proxyPass = "http://127.0.0.1:${toString cfg.port}${path}"; }; + to = path: { proxyPass = upstream path; }; in lib.listToAttrs ( map (p: lib.nameValuePair "= ${p}" (to p)) readPaths @@ -72,8 +82,8 @@ let ); in { - # What the store IS from any hive's point of view, the name it answers on and - # the port, is `swarm.victoriametrics` in ./swarm-victoriametrics-service.nix. + # What the store IS from any hive's point of view, the name it answers on, + # is `swarm.victoriametrics` in ./swarm-victoriametrics-service.nix. # `enable`, `package` and `retentionPeriod` are decisions of the host that # runs it and live under `deploy.*`. @@ -111,8 +121,7 @@ in # have. services.hyperhive.gateway.localNames = [ cfg.domain ]; - # This host serves the vhost, and the container behind it resolves - # through the hive's dnsmasq. + # This host serves the vhost, and the hive's dnsmasq answers its name. services.hyperhive.gateway.enable = lib.mkDefault true; services.hyperhive.gateway.dns.enable = lib.mkDefault true; @@ -138,7 +147,7 @@ in # follow it and POST its batch at a login page that answers 200 — # ingest reporting healthy while storing nothing. "= /opentelemetry/api/v1/push" = { - proxyPass = "http://127.0.0.1:${toString cfg.port}/opentelemetry/api/v1/push"; + proxyPass = upstream "/opentelemetry/api/v1/push"; extraConfig = '' auth_request /__metrics_push_authz; ''; @@ -153,7 +162,7 @@ in # `publishedScrapeTargets.victoriametrics` in ./swarm-otel-service.nix; # the two must agree, since that string is also the token's audience. "= /metrics" = { - proxyPass = "http://127.0.0.1:${toString cfg.port}/metrics"; + proxyPass = upstream "/metrics"; extraConfig = '' auth_request /__metrics_push_authz; ''; @@ -188,6 +197,17 @@ in }; }; + # The socket directory, created host-side before the container starts: + # nixos-container refuses to start when a bind source is missing. + # + # Root-owned and group nginx, `0770`: the store creates its socket here + # through the group (its uid is allocated inside the container, so the + # host cannot name it), and the gateway traverses it the same way. + # Everything else is kept out. + systemd.tmpfiles.rules = [ + "d ${socketDir} 0770 root ${toString nginxGid} - -" + ]; + containers.${cfg.machine} = { autoStart = true; ephemeral = false; @@ -196,31 +216,62 @@ in extraFlags = [ "--link-journal=host" ]; inherit privateNetwork; - config = - { ... }: - { - imports = [ - ../container-modules/swarm-container.nix - (import ../container-modules/swarm-container-resolver.nix { - inherit (networkCfg) bridgeIp; - dnsConsumers = [ "victoriametrics.service" ]; - }) - ]; + # A bind mount rather than a path both sides happen to know: a + # container's `/run` is an nspawn tmpfs the host cannot see into. + bindMounts.${socketDir} = { + hostPath = socketDir; + isReadOnly = false; + }; - services.hyperhive.swarmContainer = { inherit privateNetwork; }; + config = + { pkgs, ... }: + { + imports = [ ../container-modules/swarm-container.nix ]; + + # No resolver unit: with no interface but loopback there is no + # nameserver to reach, and the store resolves no names. + services.hyperhive.swarmContainer = { + inherit privateNetwork; + writesOwnResolvConf = false; + }; + + # The socket's group is the store's primary group, so that group is + # nginx's gid. A static user, not nixpkgs' DynamicUser: a dynamic + # user's group is allocated per start, and nginx is never in it. + users.groups.gateway-nginx.gid = nginxGid; + users.users.victoriametrics = { + isSystemUser = true; + group = "gateway-nginx"; + }; + + systemd.services.victoriametrics = { + serviceConfig = { + DynamicUser = lib.mkForce false; + User = "victoriametrics"; + Group = "gateway-nginx"; + # The store creates its socket under the process umask, so + # `0007` makes it `0770` and nginx's group can connect. + UMask = "0007"; + }; + # nixpkgs' readiness loop curls `http:///ping`, + # which no `unix:` address answers. + postStart = lib.mkForce '' + until ${lib.getExe pkgs.curl} -s -o /dev/null --unix-socket ${socketPath} http://localhost/ping; do + sleep 1; + done + ''; + }; services.victoriametrics = { enable = true; package = deployCfg.victoriametrics.package; retentionPeriod = deployCfg.victoriametrics.retentionPeriod; - # ⚠️ PINNED TO LOOPBACK, and this is a correction rather than a - # preference: upstream's default is `:8428`, i.e. every - # interface. The gateway is the only intended client and it is on - # this host, so binding wider would publish an unauthenticated - # write endpoint (see the OTLP note below) to whatever the host - # is reachable on. - listenAddress = "127.0.0.1:${toString cfg.port}"; + # ⚠️ A socket, not upstream's default `:8428` on every interface: + # the gateway is the only intended client, and this listener + # carries an unauthenticated write endpoint (see the OTLP note + # below). + listenAddress = "unix:${socketPath}"; }; }; }; @@ -235,8 +286,8 @@ in # deployment ever needs them. # # ⚠️ That endpoint is unauthenticated, which is why `listenAddress` above is - # loopback: the only route to it from off-host is the vhost's ingest - # location, which authenticates. The collector is still the only writer, but + # a socket only the gateway can reach: the only route to it is the vhost's + # ingest location, which authenticates. The collector is still the only writer, but # it now arrives by the swarm name rather than over loopback, because it need # not share a host with this store. } diff --git a/nix/module-eval/swarm-otel-identity.nix b/nix/module-eval/swarm-otel-identity.nix index 0172ff85..558ac020 100644 --- a/nix/module-eval/swarm-otel-identity.nix +++ b/nix/module-eval/swarm-otel-identity.nix @@ -282,7 +282,7 @@ let m = locs."= /metrics" or null; in m != null - && m.proxyPass == "http://127.0.0.1:${toString h.swarm.victoriametrics.port}/metrics" + && m.proxyPass == "http://unix:/run/swarm-victoriametrics/victoriametrics.sock:/metrics" && lib.hasInfix "auth_request /__metrics_push_authz;" m.extraConfig && !(lib.hasInfix "error_page" m.extraConfig) && lib.hasInfix "internal;" locs."= /__metrics_push_authz".extraConfig diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix index dcd92397..2b772dff 100644 --- a/nix/module-eval/swarm-services-switch.nix +++ b/nix/module-eval/swarm-services-switch.nix @@ -161,8 +161,8 @@ let lib.filter (l: swarmServiceLinkDomains ? ${l.label}) cfg.services.hyperhive.swarm.controller.links; # Every service container at once: the services-only host plus the CI - # runner. The CI runner and the review classifier are the two containers on - # a netns of their own. + # runner. The CI runner, the review classifier and the metrics store are the + # three containers on a netns of their own. serviceContainersWithCi = hive { deploy.hive-controller.enable = false; deploy.allSwarmServices = true; @@ -484,7 +484,7 @@ let let shared = lib.filter (c: !c.privateNetwork) (lib.attrValues serviceContainers); in - lib.length shared == 9 + lib.length shared == 8 && lib.all ( c: !c.config.networking.firewall.enable && !c.config.networking.resolvconf.enable ) shared; @@ -499,6 +499,44 @@ let in c.networking.firewall.enable && c.networking.resolvconf.enable; } + { + # No interface but loopback, so the socket is the store's only listener + # the host can reach. Every location that forwards to the store goes + # through it; the authelia subrequest is the one `https://` upstream. + # The read, push and scrape locations are named so an empty filter + # can't pass. + name = "the metrics store runs on a netns of its own and the gateway reaches it over its socket"; + ok = + let + host = serviceContainersWithCi; + c = host.containers.swarm-victoriametrics; + dir = "/run/swarm-victoriametrics"; + sock = "${dir}/victoriametrics.sock"; + viaSock = path: "http://unix:${sock}:${path}"; + locs = + host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.victoriametrics.domain}.locations; + toStore = lib.filter (p: p != null && !(lib.hasPrefix "https://" p)) ( + map (l: l.proxyPass) (lib.attrValues locs) + ); + unit = c.config.systemd.services.victoriametrics; + nginxGid = host.ids.gids.nginx; + in + c.privateNetwork + && c.hostBridge == null + && c.hostAddress == null + && c.config.services.victoriametrics.listenAddress == "unix:${sock}" + && locs."= /api/v1/query".proxyPass == viaSock "/api/v1/query" + && locs."= /opentelemetry/api/v1/push".proxyPass == viaSock "/opentelemetry/api/v1/push" + && locs."= /metrics".proxyPass == viaSock "/metrics" + && lib.all (lib.hasPrefix (viaSock "")) toStore + && c.bindMounts.${dir}.hostPath == dir + && builtins.elem "d ${dir} 0770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules + && c.config.users.groups.gateway-nginx.gid == nginxGid + && unit.serviceConfig.Group == "gateway-nginx" + && !unit.serviceConfig.DynamicUser + && unit.serviceConfig.UMask == "0007" + && lib.hasInfix "--unix-socket ${sock}" unit.postStart; + } { name = "a service container that sets no stateVersion of its own is on 26.05"; ok = serviceContainers.swarm-victoriametrics.config.system.stateVersion == "26.05";