swarm-victorialogs: own network namespace, gateway over a unix socket
This commit is contained in:
parent
c21b158884
commit
188dbcea8b
3 changed files with 155 additions and 76 deletions
|
|
@ -2262,10 +2262,10 @@ let
|
|||
{
|
||||
# Each old spelling lands on the new path, with its own rename warning,
|
||||
# and its value reaches the unit or vhost that reads it. Ports are chosen
|
||||
# so three of them collide on purpose: the bao collector's telemetry
|
||||
# port equals the NATS monitor port and the forge forwarder's, and the
|
||||
# NATS forwarder's equals the log store forwarder's. Each of the three
|
||||
# colocation assertions must read the moved options to see it.
|
||||
# so two of them collide on purpose: the bao collector's telemetry
|
||||
# port equals the NATS monitor port and the forge forwarder's. Each of the
|
||||
# two colocation assertions must read the moved options to see it. (The log
|
||||
# store has a netns of its own, so its forwarder's port collides with none.)
|
||||
name = "the per-service settings moved to deploy.* still apply from their old paths, with a rename warning each";
|
||||
ok =
|
||||
let
|
||||
|
|
@ -2285,7 +2285,6 @@ let
|
|||
nats.metricsPort = 17777;
|
||||
nats.otel.telemetryPort = 8897;
|
||||
otel.telemetryPort = 18889;
|
||||
victorialogs.port = 19428;
|
||||
forge.httpPort = 13000;
|
||||
matrix.allowEncryption = true;
|
||||
controller.links = [
|
||||
|
|
@ -2322,7 +2321,6 @@ let
|
|||
&& d.nats.metricsPort == 17777
|
||||
&& d.nats.otel.telemetryPort == 8897
|
||||
&& d.swarm-otel.telemetryPort == 18889
|
||||
&& d.victorialogs.port == 19428
|
||||
&& d.forgejo.httpPort == 13000
|
||||
&& d.matrix.allowEncryption
|
||||
&& d.builder.nixpkgsFlake == "path:/legacy/nixpkgs"
|
||||
|
|
@ -2343,7 +2341,6 @@ let
|
|||
old.containers.swarm-builder.config.systemd.services.swarm-builder.environment.SWARM_BUILDER_NIXPKGS_FLAKE
|
||||
== "path:/legacy/nixpkgs"
|
||||
&& (locs sw.nats.domain)."= /metrics".proxyPass == "http://127.0.0.1:17777/metrics"
|
||||
&& (locs sw.victorialogs.domain)."= /metrics".proxyPass == "http://127.0.0.1:19428/metrics"
|
||||
&& (locs sw.forge.domain)."/".proxyPass == "http://127.0.0.1:13000/"
|
||||
&& old.containers.hive-matrix.config.services.matrix-tuwunel.settings.global.allow_encryption
|
||||
&& lib.all warned [
|
||||
|
|
@ -2354,7 +2351,6 @@ let
|
|||
"swarm.nats.metricsPort"
|
||||
"swarm.nats.otel.telemetryPort"
|
||||
"swarm.otel.telemetryPort"
|
||||
"swarm.victorialogs.port"
|
||||
"swarm.forge.httpPort"
|
||||
"swarm.matrix.allowEncryption"
|
||||
"c0re.nixpkgsFlake"
|
||||
|
|
@ -2365,10 +2361,8 @@ let
|
|||
&& !(lib.any (w: lib.hasInfix "swarm.controller.queue" w) allLocal.warnings)
|
||||
&& refused old "deploy.nats.otel.telemetryPort"
|
||||
&& refused old "The forge's journal forwarder's telemetry port"
|
||||
&& refused old "The log store's journal forwarder's telemetry port"
|
||||
&& !(refused allLocal "deploy.nats.otel.telemetryPort")
|
||||
&& !(refused allLocal "The forge's journal forwarder's telemetry port")
|
||||
&& !(refused allLocal "The log store's journal forwarder's telemetry port");
|
||||
&& !(refused allLocal "The forge's journal forwarder's telemetry port");
|
||||
}
|
||||
{
|
||||
# The default render of the moved options, pinned so moving them changes
|
||||
|
|
@ -2389,7 +2383,9 @@ let
|
|||
&& h.deploy.nats.otel.telemetryPort == 8891
|
||||
&& h.deploy.swarm-otel.telemetryPort == 8889
|
||||
&& (locs h.swarm.nats.domain)."= /metrics".proxyPass == "http://127.0.0.1:7777/metrics"
|
||||
&& (locs h.swarm.victorialogs.domain)."= /metrics".proxyPass == "http://127.0.0.1:9428/metrics"
|
||||
&&
|
||||
(locs h.swarm.victorialogs.domain)."= /metrics".proxyPass
|
||||
== "http://unix:/run/swarm-victorialogs/victorialogs.sock:/metrics"
|
||||
&& (locs h.swarm.forge.domain)."/".proxyPass == "http://127.0.0.1:3000/"
|
||||
&& !allLocal.containers.hive-matrix.config.services.matrix-tuwunel.settings.global.allow_encryption
|
||||
&& env.SWARM_CONTROLLER_MATRIX_ADMIN_SYNC_IGNORE_USERS == ""
|
||||
|
|
|
|||
|
|
@ -511,7 +511,7 @@ let
|
|||
in
|
||||
h.deploy.victorialogs.enable
|
||||
&& locs ? "= /metrics"
|
||||
&& locs."= /metrics".proxyPass == "http://127.0.0.1:${toString h.deploy.victorialogs.port}/metrics"
|
||||
&& locs."= /metrics".proxyPass == "http://unix:/run/swarm-victorialogs/victorialogs.sock:/metrics"
|
||||
&& checksOf locs."= /metrics" == [ "auth_request /__hive_authelia;" ]
|
||||
&& !(lib.hasInfix "error_page" locs."= /metrics".extraConfig)
|
||||
&& lib.hasInfix "error_page 401 =302" locs."/".extraConfig
|
||||
|
|
|
|||
|
|
@ -843,14 +843,14 @@ let
|
|||
}
|
||||
{
|
||||
# Both values come from ../container-modules/swarm-container.nix.
|
||||
# Four of the 16 service containers share the host netns (hive-forge, bao,
|
||||
# the queue, the log store) and twelve have one of their own.
|
||||
# Three of the 16 service containers share the host netns (hive-forge, bao,
|
||||
# the queue) and thirteen have one of their own.
|
||||
name = "a service container on the host netns runs no firewall or resolvconf of its own";
|
||||
ok =
|
||||
let
|
||||
shared = lib.filter (c: !c.privateNetwork) (lib.attrValues serviceContainers);
|
||||
in
|
||||
lib.length shared == 4
|
||||
lib.length shared == 3
|
||||
&& lib.all (
|
||||
c: !c.config.networking.firewall.enable && !c.config.networking.resolvconf.enable
|
||||
) shared;
|
||||
|
|
@ -990,6 +990,50 @@ let
|
|||
&& bridge.serviceConfig.Group == "gateway-nginx"
|
||||
&& lib.elem dir unit.serviceConfig.ReadWritePaths;
|
||||
}
|
||||
{
|
||||
# On the bridge by DHCP, listening on a socket only the gateway's group can
|
||||
# reach, so the store binds no host port. Every location that forwards to
|
||||
# it goes through the socket (the authelia subrequests are the `https://`
|
||||
# upstream); the read, ingest, scrape and catch-all locations are named so
|
||||
# an empty filter can't pass, and the old loopback port is named nowhere.
|
||||
name = "the log store runs on a netns of its own and the gateway reaches it over its socket";
|
||||
ok =
|
||||
let
|
||||
host = serviceContainersWithCi;
|
||||
net = host.services.hyperhive.network;
|
||||
c = host.containers.swarm-victorialogs;
|
||||
dir = "/run/swarm-victorialogs";
|
||||
sock = "${dir}/victorialogs.sock";
|
||||
viaSock = path: "http://unix:${sock}:${path}";
|
||||
locs =
|
||||
host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.victorialogs.domain}.locations;
|
||||
toStore = lib.filter (
|
||||
p: p != null && !(lib.hasPrefix "https://" p) && !(lib.hasPrefix "http://unix:/run/swarm-logs" p)
|
||||
) (map (l: l.proxyPass) (lib.attrValues locs));
|
||||
unit = c.config.systemd.services.victorialogs;
|
||||
nginxGid = host.ids.gids.nginx;
|
||||
in
|
||||
c.privateNetwork
|
||||
&& c.hostBridge == net.bridgeName
|
||||
&& c.hostAddress == null
|
||||
&& c.localAddress == null
|
||||
&& c.config.networking.interfaces.eth0.useDHCP
|
||||
&& c.config.services.victorialogs.listenAddress == "unix:${sock}"
|
||||
&& locs."/".proxyPass == viaSock "/"
|
||||
&& locs."= /insert/opentelemetry/v1/logs".proxyPass == viaSock "/insert/opentelemetry/v1/logs"
|
||||
&& locs."^~ /select/logsql/".proxyPass == "http://unix:${sock}"
|
||||
&& locs."= /metrics".proxyPass == viaSock "/metrics"
|
||||
&& toStore != [ ]
|
||||
&& lib.all (lib.hasPrefix "http://unix:${sock}") toStore
|
||||
&& !(lib.any (lib.hasInfix "127.0.0.1") toStore)
|
||||
&& c.bindMounts.${dir}.hostPath == dir
|
||||
&& builtins.elem "d ${dir} 0770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules
|
||||
&& c.config.users.groups.gateway-nginx.gid == nginxGid
|
||||
&& unit.serviceConfig.Group == "gateway-nginx"
|
||||
&& !unit.serviceConfig.DynamicUser
|
||||
&& unit.serviceConfig.UMask == "0007"
|
||||
&& lib.hasInfix "--unix-socket ${sock}" unit.postStart;
|
||||
}
|
||||
{
|
||||
# On the bridge by DHCP, listening on a socket (protocol `socket` ignores the
|
||||
# upstream-default `http_port`, which stays in the settings): the
|
||||
|
|
|
|||
Loading…
Reference in a new issue