diff --git a/docs/4-implementation/swarm/telemetry.md b/docs/4-implementation/swarm/telemetry.md index fbbc60bae..1c1f596bf 100644 --- a/docs/4-implementation/swarm/telemetry.md +++ b/docs/4-implementation/swarm/telemetry.md @@ -188,10 +188,12 @@ swarm. | `deploy.victorialogs.authVerdictTtl` | Default `60` seconds that `swarm-logs-auth` keeps granting an agent secret it verified, without asking the secret store. A replaced or deleted secret keeps working at the log store for up to that long. A revoked `logs` permission does too. | | `deploy.victorialogs.grafanaAuthPackage` | The `swarm-metrics-auth` build that runs as `swarm-logs-grafana-auth`. Set on its own: `deploy.victoriametrics.authPackage` doesn't reach it. | | `swarm.victorialogs.domain` | Only to rename it. | -| `deploy.victorialogs.port` | Only if something else on the services host already claims `9428`. | -It binds loopback and takes no credential of its own: the gateway vhost is -the only way in, and the collector is the only intended writer. +It listens on a unix socket in a network namespace of its own, on the hive +bridge so its journal forwarder can reach the swarm collector, and binds no host +port. Only the gateway's nginx can open the socket. It takes no credential of +its own: the gateway vhost is the only way in, and the collector is the only +intended writer. **Who may query.** `/select/logsql/` takes one of three credentials. An agent presents its `swarm-agent..` bearer, checked by diff --git a/nix/host-modules/deploy.nix b/nix/host-modules/deploy.nix index d415e158b..85723e242 100644 --- a/nix/host-modules/deploy.nix +++ b/nix/host-modules/deploy.nix @@ -85,6 +85,16 @@ let in { imports = [ + (lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "victorialogs" "port" ] '' + VictoriaLogs listens on a unix socket in a network namespace of its own, + and the gateway reaches it there, so it binds no host port. Remove this + definition. + '') + (lib.mkRemovedOptionModule [ "services" "hyperhive" "deploy" "victorialogs" "port" ] '' + VictoriaLogs listens on a unix socket in a network namespace of its own, + and the gateway reaches it there, so it binds no host port. Remove this + definition. + '') (lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "victoriametrics" "port" ] '' VictoriaMetrics listens on a unix socket in a network namespace of its own, and the gateway reaches it there, so it binds no host port. Remove @@ -517,10 +527,6 @@ in [ "services" "hyperhive" "swarm" "otel" "telemetryPort" ] [ "services" "hyperhive" "deploy" "swarm-otel" "telemetryPort" ] ) - (lib.mkRenamedOptionModule - [ "services" "hyperhive" "swarm" "victorialogs" "port" ] - [ "services" "hyperhive" "deploy" "victorialogs" "port" ] - ) (lib.mkRenamedOptionModule [ "services" "hyperhive" "swarm" "forge" "httpPort" ] [ "services" "hyperhive" "deploy" "forgejo" "httpPort" ] diff --git a/nix/host-modules/swarm-victorialogs.nix b/nix/host-modules/swarm-victorialogs.nix index fbcfc7f01..4ef88a29a 100644 --- a/nix/host-modules/swarm-victorialogs.nix +++ b/nix/host-modules/swarm-victorialogs.nix @@ -17,8 +17,8 @@ # below, same shape as `swarm-ui.nix`'s) — see that file's copy for the full # rationale (forceSSL is load-bearing there too: authelia answers a plain-http # auth subrequest with 400, which `auth_request` cannot interpret as anything -# but a broken check). The store's own listener stays loopback-only and -# unauthenticated, and the vhost is now the ONLY way in from outside: the +# but a broken check). The store's own listener is a unix socket in a +# netns of its own and is unauthenticated, and the vhost is the ONLY way in: the # collector pushes through it too, at its own `= /insert/...` location, because # a swarm has one log store and the collector need not share a host with it. # ⚠️ That ingest location deliberately does not carry `swarmAuthRequest` — a @@ -100,9 +100,23 @@ let grafanaAuthCredential = id: "/run/credentials/${grafanaAuthUnit}.service/${id}"; grafanaBearerStorePath = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer"; - # Shared host netns, like every sibling swarm container: Grafana reaches - # this at 127.0.0.1:. - privateNetwork = false; + # A netns of its own on the hive bridge, as the cache's container has + # (./swarm-attic.nix): the forwarder below dials the swarm collector, authelia + # and the store by name, so the container keeps an interface and the bridge's + # resolver. The store listens on a unix socket only, so no host port is + # bound. Grafana reaches the store by its domain through the gateway. + privateNetwork = true; + + # The unix socket the gateway's nginx connects to. The directory's mode is + # what keeps it private, so it holds this one socket. + socketDir = "/run/swarm-victorialogs"; + socketPath = "${socketDir}/victorialogs.sock"; + socketGroup = "gateway-nginx"; + # Only the number crosses into the container, which knows the group by name. + nginxGid = config.ids.gids.nginx; + # The upstream every location below forwards to. A location that adds a path + # writes `:`; one with no path leaves the request's own. + upstream = "http://unix:${socketPath}"; # The container's journal forwarder pushes to its own receiver on the swarm # collector, the route `serviceProducers.swarm-victorialogs` in @@ -110,8 +124,7 @@ let forwarderClientId = cfg.otel.clientId; forwarderEndpoint = "https://${hyperhiveCfg.swarm.otel.domain}/${cfg.machine}"; forwarderUnit = "swarm-journal-forwarder-oidc"; - # In the host netns: distinct from every other collector's there, which the - # assertion below checks. + # Loopback, in the container's own netns. forwarderTelemetryPort = 8897; forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}"; # The responder's store identity, handed into the container as system @@ -134,7 +147,7 @@ in { # What the store IS from any hive's point of view, the name it answers on, is # `swarm.victorialogs` in ./swarm-victorialogs-service.nix. - # `enable`, `package`, `port` and `retentionPeriod` are decisions of the host that + # `enable`, `package` and `retentionPeriod` are decisions of the host that # runs it and live under `deploy.*`. # Retention is a property of the store this host runs, not something the @@ -149,22 +162,6 @@ in description = "VictoriaLogs package to run."; }; - options.services.hyperhive.deploy.victorialogs.port = lib.mkOption { - type = lib.types.port; - default = 9428; - description = '' - Loopback port the store listens on. Upstream's default, kept - because there is no reason to move it and a familiar number is - one less thing an operator has to look up. - - ⚠️ Every swarm container shares the host's network namespace, so - this is a swarm-wide claim rather than a per-container one — two - modules picking the same number collide at runtime with no bind - error and nothing in any log. No repo-wide - check catches it. - ''; - }; - options.services.hyperhive.deploy.victorialogs.retentionPeriod = lib.mkOption { type = lib.types.str; default = "30d"; @@ -310,20 +307,6 @@ in Declare it in swarm-otel-service.nix. ''; } - { - assertion = - !(lib.elem forwarderTelemetryPort [ - hyperhiveCfg.otel.telemetryPort - hyperhiveCfg.deploy.bao.otel.telemetryPort - hyperhiveCfg.deploy.nats.otel.telemetryPort - ]); - message = '' - The log store's journal forwarder's telemetry port (${toString forwarderTelemetryPort}) - collides with another collector's in the host netns: the hive - collector's, the store forwarder's or the - queue forwarder's. - ''; - } ]; # nspawn fails to start the container when a credential source is @@ -375,9 +358,9 @@ in services.hyperhive.gateway.enable = lib.mkDefault true; services.hyperhive.gateway.dns.enable = lib.mkDefault true; - # Authenticated front door onto the loopback-only store — see the + # Authenticated front door onto the store's socket — see the # file-top comment for why this is safe to add without touching the - # store's own (still unauthenticated, still loopback) listener at all. + # store's own (still unauthenticated, socket-only) listener at all. # `forceSSL`: same asymmetry `swarm-ui.nix` documents — # authelia answers a plain-http auth subrequest with 400, which # `auth_request` cannot read as anything but a broken check, so this @@ -387,7 +370,7 @@ in forceSSL = true; locations = { "/" = { - proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/"; + proxyPass = "${upstream}:/"; extraConfig = swarmAuthRequest; }; @@ -406,7 +389,7 @@ in # note on `swarmAuthRequest` above), so naming it here is required # rather than redundant. "= /insert/opentelemetry/v1/logs" = { - proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/insert/opentelemetry/v1/logs"; + proxyPass = "${upstream}:/insert/opentelemetry/v1/logs"; extraConfig = '' auth_request /__swarm_logs_ingest_authz; ''; @@ -440,7 +423,7 @@ in "^~ /select/logsql/" = { # No URI part, so the request path and its query string reach # the store as the caller sent them. - proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}"; + proxyPass = upstream; extraConfig = '' auth_request /__swarm_logs_auth; ''; @@ -457,7 +440,7 @@ in # authelia's rules for this URL admit the collector's client and # deny every other subject (see `metricsRules` in ./swarm-authelia.nix). "= /metrics" = { - proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/metrics"; + proxyPass = "${upstream}:/metrics"; extraConfig = '' auth_request /__hive_authelia; ''; @@ -659,6 +642,12 @@ in }; }; + # The socket directory is the gateway's way in: its group is nginx's. + systemd.tmpfiles.rules = [ "d ${socketDir} 0770 root ${toString nginxGid} - -" ]; + + # The hive bridge carries the container's DHCP lease and its resolver. + services.hyperhive.network.enable = lib.mkDefault true; + containers.${cfg.machine} = { autoStart = true; ephemeral = false; @@ -668,10 +657,20 @@ in id: source: "--load-credential=${id}:${source}" ) forwarderStoreFiles; inherit privateNetwork; - bindMounts = caTrust.bindMount; + hostBridge = networkCfg.bridgeName; + # A bind mount, not a path both sides know: a container's `/run` is an + # nspawn tmpfs the host cannot see into. Connecting to a socket is a + # write to it. + bindMounts = { + ${socketDir} = { + hostPath = socketDir; + isReadOnly = false; + }; + } + // caTrust.bindMount; config = - { ... }: + { pkgs, ... }: { imports = [ ../container-modules/swarm-container.nix @@ -697,6 +696,37 @@ in services.hyperhive.swarmContainer = { inherit privateNetwork; }; + # The bridge address comes from the gateway's DHCP. `swarm-bridge-dns` + # owns resolv.conf, so dhcpcd must not write it over that unit. + networking.interfaces.eth0.useDHCP = true; + networking.dhcpcd.extraConfig = "nohook resolv.conf"; + + # The socket's group is the store's primary group, so that group is + # nginx's gid. A static user, not nixpkgs' DynamicUser: a dynamic + # user's group is allocated per start, and nginx is never in it. + users.groups.${socketGroup}.gid = nginxGid; + users.users.victorialogs = { + isSystemUser = true; + group = socketGroup; + }; + systemd.services.victorialogs = { + serviceConfig = { + DynamicUser = lib.mkForce false; + User = "victorialogs"; + Group = socketGroup; + # The store creates its socket under the process umask, so `0007` + # makes it `0770` and nginx's group can connect. + UMask = "0007"; + }; + # nixpkgs' readiness loop curls a TCP URL, which no `unix:` address + # answers. + postStart = lib.mkForce '' + until ${lib.getExe pkgs.curl} -s -o /dev/null --unix-socket ${socketPath} http://localhost/ping; do + sleep 1; + done + ''; + }; + # Authenticated as the forwarder's own OIDC client, with the # responder's store identity: one identity for the whole container. # It is a system credential, so the fetch unit loads it. @@ -722,12 +752,13 @@ in enable = true; package = deployCfg.victorialogs.package; - # ⚠️ PINNED TO LOOPBACK: upstream's default listens on every - # interface, and this endpoint accepts writes as well as reads. - # Nothing authenticates it either — upstream offers one basic-auth - # pair (`-httpAuth.username` / `-httpAuth.password`) and this - # module sets neither. The bind address is the boundary. - listenAddress = "127.0.0.1:${toString deployCfg.victorialogs.port}"; + # ⚠️ A SOCKET, not upstream's default of every interface: this + # endpoint accepts writes as well as reads. Nothing authenticates it + # either — upstream offers one basic-auth pair + # (`-httpAuth.username` / `-httpAuth.password`) and this module sets + # neither. The socket's directory mode is the boundary: only the + # gateway's group can connect. + listenAddress = "unix:${socketPath}"; extraOptions = [ "-retentionPeriod=${deployCfg.victorialogs.retentionPeriod}" ]; }; @@ -749,7 +780,7 @@ in # code is not evidence. # # ⚠️ Like the metrics store's, that endpoint is unauthenticated — which is - # why `listenAddress` above is loopback and why nothing reaches it except + # why `listenAddress` above is a socket and why nothing reaches it except # through the gateway, where the ingest route is authenticated. (This module # declares a vhost.) } diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index 029ce1f09..0faef703f 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -2262,10 +2262,10 @@ let { # Each old spelling lands on the new path, with its own rename warning, # and its value reaches the unit or vhost that reads it. Ports are chosen - # so three of them collide on purpose: the bao collector's telemetry - # port equals the NATS monitor port and the forge forwarder's, and the - # NATS forwarder's equals the log store forwarder's. Each of the three - # colocation assertions must read the moved options to see it. + # so two of them collide on purpose: the bao collector's telemetry + # port equals the NATS monitor port and the forge forwarder's. Each of the + # two colocation assertions must read the moved options to see it. (The log + # store has a netns of its own, so its forwarder's port collides with none.) name = "the per-service settings moved to deploy.* still apply from their old paths, with a rename warning each"; ok = let @@ -2285,7 +2285,6 @@ let nats.metricsPort = 17777; nats.otel.telemetryPort = 8897; otel.telemetryPort = 18889; - victorialogs.port = 19428; forge.httpPort = 13000; matrix.allowEncryption = true; controller.links = [ @@ -2322,7 +2321,6 @@ let && d.nats.metricsPort == 17777 && d.nats.otel.telemetryPort == 8897 && d.swarm-otel.telemetryPort == 18889 - && d.victorialogs.port == 19428 && d.forgejo.httpPort == 13000 && d.matrix.allowEncryption && d.builder.nixpkgsFlake == "path:/legacy/nixpkgs" @@ -2343,7 +2341,6 @@ let old.containers.swarm-builder.config.systemd.services.swarm-builder.environment.SWARM_BUILDER_NIXPKGS_FLAKE == "path:/legacy/nixpkgs" && (locs sw.nats.domain)."= /metrics".proxyPass == "http://127.0.0.1:17777/metrics" - && (locs sw.victorialogs.domain)."= /metrics".proxyPass == "http://127.0.0.1:19428/metrics" && (locs sw.forge.domain)."/".proxyPass == "http://127.0.0.1:13000/" && old.containers.hive-matrix.config.services.matrix-tuwunel.settings.global.allow_encryption && lib.all warned [ @@ -2354,7 +2351,6 @@ let "swarm.nats.metricsPort" "swarm.nats.otel.telemetryPort" "swarm.otel.telemetryPort" - "swarm.victorialogs.port" "swarm.forge.httpPort" "swarm.matrix.allowEncryption" "c0re.nixpkgsFlake" @@ -2365,10 +2361,8 @@ let && !(lib.any (w: lib.hasInfix "swarm.controller.queue" w) allLocal.warnings) && refused old "deploy.nats.otel.telemetryPort" && refused old "The forge's journal forwarder's telemetry port" - && refused old "The log store's journal forwarder's telemetry port" && !(refused allLocal "deploy.nats.otel.telemetryPort") - && !(refused allLocal "The forge's journal forwarder's telemetry port") - && !(refused allLocal "The log store's journal forwarder's telemetry port"); + && !(refused allLocal "The forge's journal forwarder's telemetry port"); } { # The default render of the moved options, pinned so moving them changes @@ -2389,7 +2383,9 @@ let && h.deploy.nats.otel.telemetryPort == 8891 && h.deploy.swarm-otel.telemetryPort == 8889 && (locs h.swarm.nats.domain)."= /metrics".proxyPass == "http://127.0.0.1:7777/metrics" - && (locs h.swarm.victorialogs.domain)."= /metrics".proxyPass == "http://127.0.0.1:9428/metrics" + && + (locs h.swarm.victorialogs.domain)."= /metrics".proxyPass + == "http://unix:/run/swarm-victorialogs/victorialogs.sock:/metrics" && (locs h.swarm.forge.domain)."/".proxyPass == "http://127.0.0.1:3000/" && !allLocal.containers.hive-matrix.config.services.matrix-tuwunel.settings.global.allow_encryption && env.SWARM_CONTROLLER_MATRIX_ADMIN_SYNC_IGNORE_USERS == "" diff --git a/nix/module-eval/swarm-otel-identity.nix b/nix/module-eval/swarm-otel-identity.nix index d1b110e70..1db652df3 100644 --- a/nix/module-eval/swarm-otel-identity.nix +++ b/nix/module-eval/swarm-otel-identity.nix @@ -511,7 +511,7 @@ let in h.deploy.victorialogs.enable && locs ? "= /metrics" - && locs."= /metrics".proxyPass == "http://127.0.0.1:${toString h.deploy.victorialogs.port}/metrics" + && locs."= /metrics".proxyPass == "http://unix:/run/swarm-victorialogs/victorialogs.sock:/metrics" && checksOf locs."= /metrics" == [ "auth_request /__hive_authelia;" ] && !(lib.hasInfix "error_page" locs."= /metrics".extraConfig) && lib.hasInfix "error_page 401 =302" locs."/".extraConfig diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix index 3cf6330f6..ed09864b8 100644 --- a/nix/module-eval/swarm-services-switch.nix +++ b/nix/module-eval/swarm-services-switch.nix @@ -843,14 +843,14 @@ let } { # Both values come from ../container-modules/swarm-container.nix. - # Four of the 16 service containers share the host netns (hive-forge, bao, - # the queue, the log store) and twelve have one of their own. + # Three of the 16 service containers share the host netns (hive-forge, bao, + # the queue) and thirteen have one of their own. name = "a service container on the host netns runs no firewall or resolvconf of its own"; ok = let shared = lib.filter (c: !c.privateNetwork) (lib.attrValues serviceContainers); in - lib.length shared == 4 + lib.length shared == 3 && lib.all ( c: !c.config.networking.firewall.enable && !c.config.networking.resolvconf.enable ) shared; @@ -990,6 +990,50 @@ let && bridge.serviceConfig.Group == "gateway-nginx" && lib.elem dir unit.serviceConfig.ReadWritePaths; } + { + # On the bridge by DHCP, listening on a socket only the gateway's group can + # reach, so the store binds no host port. Every location that forwards to + # it goes through the socket (the authelia subrequests are the `https://` + # upstream); the read, ingest, scrape and catch-all locations are named so + # an empty filter can't pass, and the old loopback port is named nowhere. + name = "the log store runs on a netns of its own and the gateway reaches it over its socket"; + ok = + let + host = serviceContainersWithCi; + net = host.services.hyperhive.network; + c = host.containers.swarm-victorialogs; + dir = "/run/swarm-victorialogs"; + sock = "${dir}/victorialogs.sock"; + viaSock = path: "http://unix:${sock}:${path}"; + locs = + host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.victorialogs.domain}.locations; + toStore = lib.filter ( + p: p != null && !(lib.hasPrefix "https://" p) && !(lib.hasPrefix "http://unix:/run/swarm-logs" p) + ) (map (l: l.proxyPass) (lib.attrValues locs)); + unit = c.config.systemd.services.victorialogs; + nginxGid = host.ids.gids.nginx; + in + c.privateNetwork + && c.hostBridge == net.bridgeName + && c.hostAddress == null + && c.localAddress == null + && c.config.networking.interfaces.eth0.useDHCP + && c.config.services.victorialogs.listenAddress == "unix:${sock}" + && locs."/".proxyPass == viaSock "/" + && locs."= /insert/opentelemetry/v1/logs".proxyPass == viaSock "/insert/opentelemetry/v1/logs" + && locs."^~ /select/logsql/".proxyPass == "http://unix:${sock}" + && locs."= /metrics".proxyPass == viaSock "/metrics" + && toStore != [ ] + && lib.all (lib.hasPrefix "http://unix:${sock}") toStore + && !(lib.any (lib.hasInfix "127.0.0.1") toStore) + && c.bindMounts.${dir}.hostPath == dir + && builtins.elem "d ${dir} 0770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules + && c.config.users.groups.gateway-nginx.gid == nginxGid + && unit.serviceConfig.Group == "gateway-nginx" + && !unit.serviceConfig.DynamicUser + && unit.serviceConfig.UMask == "0007" + && lib.hasInfix "--unix-socket ${sock}" unit.postStart; + } { # On the bridge by DHCP, listening on a socket (protocol `socket` ignores the # upstream-default `http_port`, which stays in the settings): the