swarm-victorialogs: own network namespace, gateway over a unix socket
This commit is contained in:
parent
c21b158884
commit
188dbcea8b
2 changed files with 155 additions and 76 deletions
|
|
@ -85,6 +85,16 @@ let
|
|||
in
|
||||
{
|
||||
imports = [
|
||||
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "victorialogs" "port" ] ''
|
||||
VictoriaLogs listens on a unix socket in a network namespace of its own,
|
||||
and the gateway reaches it there, so it binds no host port. Remove this
|
||||
definition.
|
||||
'')
|
||||
(lib.mkRemovedOptionModule [ "services" "hyperhive" "deploy" "victorialogs" "port" ] ''
|
||||
VictoriaLogs listens on a unix socket in a network namespace of its own,
|
||||
and the gateway reaches it there, so it binds no host port. Remove this
|
||||
definition.
|
||||
'')
|
||||
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "victoriametrics" "port" ] ''
|
||||
VictoriaMetrics listens on a unix socket in a network namespace of its
|
||||
own, and the gateway reaches it there, so it binds no host port. Remove
|
||||
|
|
@ -517,10 +527,6 @@ in
|
|||
[ "services" "hyperhive" "swarm" "otel" "telemetryPort" ]
|
||||
[ "services" "hyperhive" "deploy" "swarm-otel" "telemetryPort" ]
|
||||
)
|
||||
(lib.mkRenamedOptionModule
|
||||
[ "services" "hyperhive" "swarm" "victorialogs" "port" ]
|
||||
[ "services" "hyperhive" "deploy" "victorialogs" "port" ]
|
||||
)
|
||||
(lib.mkRenamedOptionModule
|
||||
[ "services" "hyperhive" "swarm" "forge" "httpPort" ]
|
||||
[ "services" "hyperhive" "deploy" "forgejo" "httpPort" ]
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@
|
|||
# below, same shape as `swarm-ui.nix`'s) — see that file's copy for the full
|
||||
# rationale (forceSSL is load-bearing there too: authelia answers a plain-http
|
||||
# auth subrequest with 400, which `auth_request` cannot interpret as anything
|
||||
# but a broken check). The store's own listener stays loopback-only and
|
||||
# unauthenticated, and the vhost is now the ONLY way in from outside: the
|
||||
# but a broken check). The store's own listener is a unix socket in a
|
||||
# netns of its own and is unauthenticated, and the vhost is the ONLY way in: the
|
||||
# collector pushes through it too, at its own `= /insert/...` location, because
|
||||
# a swarm has one log store and the collector need not share a host with it.
|
||||
# ⚠️ That ingest location deliberately does not carry `swarmAuthRequest` — a
|
||||
|
|
@ -100,9 +100,23 @@ let
|
|||
grafanaAuthCredential = id: "/run/credentials/${grafanaAuthUnit}.service/${id}";
|
||||
grafanaBearerStorePath = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer";
|
||||
|
||||
# Shared host netns, like every sibling swarm container: Grafana reaches
|
||||
# this at 127.0.0.1:<port>.
|
||||
privateNetwork = false;
|
||||
# A netns of its own on the hive bridge, as the cache's container has
|
||||
# (./swarm-attic.nix): the forwarder below dials the swarm collector, authelia
|
||||
# and the store by name, so the container keeps an interface and the bridge's
|
||||
# resolver. The store listens on a unix socket only, so no host port is
|
||||
# bound. Grafana reaches the store by its domain through the gateway.
|
||||
privateNetwork = true;
|
||||
|
||||
# The unix socket the gateway's nginx connects to. The directory's mode is
|
||||
# what keeps it private, so it holds this one socket.
|
||||
socketDir = "/run/swarm-victorialogs";
|
||||
socketPath = "${socketDir}/victorialogs.sock";
|
||||
socketGroup = "gateway-nginx";
|
||||
# Only the number crosses into the container, which knows the group by name.
|
||||
nginxGid = config.ids.gids.nginx;
|
||||
# The upstream every location below forwards to. A location that adds a path
|
||||
# writes `<upstream>:<path>`; one with no path leaves the request's own.
|
||||
upstream = "http://unix:${socketPath}";
|
||||
|
||||
# The container's journal forwarder pushes to its own receiver on the swarm
|
||||
# collector, the route `serviceProducers.swarm-victorialogs` in
|
||||
|
|
@ -110,8 +124,7 @@ let
|
|||
forwarderClientId = cfg.otel.clientId;
|
||||
forwarderEndpoint = "https://${hyperhiveCfg.swarm.otel.domain}/${cfg.machine}";
|
||||
forwarderUnit = "swarm-journal-forwarder-oidc";
|
||||
# In the host netns: distinct from every other collector's there, which the
|
||||
# assertion below checks.
|
||||
# Loopback, in the container's own netns.
|
||||
forwarderTelemetryPort = 8897;
|
||||
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
|
||||
# The responder's store identity, handed into the container as system
|
||||
|
|
@ -134,7 +147,7 @@ in
|
|||
{
|
||||
# What the store IS from any hive's point of view, the name it answers on, is
|
||||
# `swarm.victorialogs` in ./swarm-victorialogs-service.nix.
|
||||
# `enable`, `package`, `port` and `retentionPeriod` are decisions of the host that
|
||||
# `enable`, `package` and `retentionPeriod` are decisions of the host that
|
||||
# runs it and live under `deploy.*`.
|
||||
|
||||
# Retention is a property of the store this host runs, not something the
|
||||
|
|
@ -149,22 +162,6 @@ in
|
|||
description = "VictoriaLogs package to run.";
|
||||
};
|
||||
|
||||
options.services.hyperhive.deploy.victorialogs.port = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 9428;
|
||||
description = ''
|
||||
Loopback port the store listens on. Upstream's default, kept
|
||||
because there is no reason to move it and a familiar number is
|
||||
one less thing an operator has to look up.
|
||||
|
||||
⚠️ Every swarm container shares the host's network namespace, so
|
||||
this is a swarm-wide claim rather than a per-container one — two
|
||||
modules picking the same number collide at runtime with no bind
|
||||
error and nothing in any log. No repo-wide
|
||||
check catches it.
|
||||
'';
|
||||
};
|
||||
|
||||
options.services.hyperhive.deploy.victorialogs.retentionPeriod = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "30d";
|
||||
|
|
@ -310,20 +307,6 @@ in
|
|||
Declare it in swarm-otel-service.nix.
|
||||
'';
|
||||
}
|
||||
{
|
||||
assertion =
|
||||
!(lib.elem forwarderTelemetryPort [
|
||||
hyperhiveCfg.otel.telemetryPort
|
||||
hyperhiveCfg.deploy.bao.otel.telemetryPort
|
||||
hyperhiveCfg.deploy.nats.otel.telemetryPort
|
||||
]);
|
||||
message = ''
|
||||
The log store's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
|
||||
collides with another collector's in the host netns: the hive
|
||||
collector's, the store forwarder's or the
|
||||
queue forwarder's.
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
# nspawn fails to start the container when a credential source is
|
||||
|
|
@ -375,9 +358,9 @@ in
|
|||
services.hyperhive.gateway.enable = lib.mkDefault true;
|
||||
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
||||
|
||||
# Authenticated front door onto the loopback-only store — see the
|
||||
# Authenticated front door onto the store's socket — see the
|
||||
# file-top comment for why this is safe to add without touching the
|
||||
# store's own (still unauthenticated, still loopback) listener at all.
|
||||
# store's own (still unauthenticated, socket-only) listener at all.
|
||||
# `forceSSL`: same asymmetry `swarm-ui.nix` documents —
|
||||
# authelia answers a plain-http auth subrequest with 400, which
|
||||
# `auth_request` cannot read as anything but a broken check, so this
|
||||
|
|
@ -387,7 +370,7 @@ in
|
|||
forceSSL = true;
|
||||
locations = {
|
||||
"/" = {
|
||||
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/";
|
||||
proxyPass = "${upstream}:/";
|
||||
extraConfig = swarmAuthRequest;
|
||||
};
|
||||
|
||||
|
|
@ -406,7 +389,7 @@ in
|
|||
# note on `swarmAuthRequest` above), so naming it here is required
|
||||
# rather than redundant.
|
||||
"= /insert/opentelemetry/v1/logs" = {
|
||||
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/insert/opentelemetry/v1/logs";
|
||||
proxyPass = "${upstream}:/insert/opentelemetry/v1/logs";
|
||||
extraConfig = ''
|
||||
auth_request /__swarm_logs_ingest_authz;
|
||||
'';
|
||||
|
|
@ -440,7 +423,7 @@ in
|
|||
"^~ /select/logsql/" = {
|
||||
# No URI part, so the request path and its query string reach
|
||||
# the store as the caller sent them.
|
||||
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}";
|
||||
proxyPass = upstream;
|
||||
extraConfig = ''
|
||||
auth_request /__swarm_logs_auth;
|
||||
'';
|
||||
|
|
@ -457,7 +440,7 @@ in
|
|||
# authelia's rules for this URL admit the collector's client and
|
||||
# deny every other subject (see `metricsRules` in ./swarm-authelia.nix).
|
||||
"= /metrics" = {
|
||||
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/metrics";
|
||||
proxyPass = "${upstream}:/metrics";
|
||||
extraConfig = ''
|
||||
auth_request /__hive_authelia;
|
||||
'';
|
||||
|
|
@ -659,6 +642,12 @@ in
|
|||
};
|
||||
};
|
||||
|
||||
# The socket directory is the gateway's way in: its group is nginx's.
|
||||
systemd.tmpfiles.rules = [ "d ${socketDir} 0770 root ${toString nginxGid} - -" ];
|
||||
|
||||
# The hive bridge carries the container's DHCP lease and its resolver.
|
||||
services.hyperhive.network.enable = lib.mkDefault true;
|
||||
|
||||
containers.${cfg.machine} = {
|
||||
autoStart = true;
|
||||
ephemeral = false;
|
||||
|
|
@ -668,10 +657,20 @@ in
|
|||
id: source: "--load-credential=${id}:${source}"
|
||||
) forwarderStoreFiles;
|
||||
inherit privateNetwork;
|
||||
bindMounts = caTrust.bindMount;
|
||||
hostBridge = networkCfg.bridgeName;
|
||||
# A bind mount, not a path both sides know: a container's `/run` is an
|
||||
# nspawn tmpfs the host cannot see into. Connecting to a socket is a
|
||||
# write to it.
|
||||
bindMounts = {
|
||||
${socketDir} = {
|
||||
hostPath = socketDir;
|
||||
isReadOnly = false;
|
||||
};
|
||||
}
|
||||
// caTrust.bindMount;
|
||||
|
||||
config =
|
||||
{ ... }:
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
imports = [
|
||||
../container-modules/swarm-container.nix
|
||||
|
|
@ -697,6 +696,37 @@ in
|
|||
|
||||
services.hyperhive.swarmContainer = { inherit privateNetwork; };
|
||||
|
||||
# The bridge address comes from the gateway's DHCP. `swarm-bridge-dns`
|
||||
# owns resolv.conf, so dhcpcd must not write it over that unit.
|
||||
networking.interfaces.eth0.useDHCP = true;
|
||||
networking.dhcpcd.extraConfig = "nohook resolv.conf";
|
||||
|
||||
# The socket's group is the store's primary group, so that group is
|
||||
# nginx's gid. A static user, not nixpkgs' DynamicUser: a dynamic
|
||||
# user's group is allocated per start, and nginx is never in it.
|
||||
users.groups.${socketGroup}.gid = nginxGid;
|
||||
users.users.victorialogs = {
|
||||
isSystemUser = true;
|
||||
group = socketGroup;
|
||||
};
|
||||
systemd.services.victorialogs = {
|
||||
serviceConfig = {
|
||||
DynamicUser = lib.mkForce false;
|
||||
User = "victorialogs";
|
||||
Group = socketGroup;
|
||||
# The store creates its socket under the process umask, so `0007`
|
||||
# makes it `0770` and nginx's group can connect.
|
||||
UMask = "0007";
|
||||
};
|
||||
# nixpkgs' readiness loop curls a TCP URL, which no `unix:` address
|
||||
# answers.
|
||||
postStart = lib.mkForce ''
|
||||
until ${lib.getExe pkgs.curl} -s -o /dev/null --unix-socket ${socketPath} http://localhost/ping; do
|
||||
sleep 1;
|
||||
done
|
||||
'';
|
||||
};
|
||||
|
||||
# Authenticated as the forwarder's own OIDC client, with the
|
||||
# responder's store identity: one identity for the whole container.
|
||||
# It is a system credential, so the fetch unit loads it.
|
||||
|
|
@ -722,12 +752,13 @@ in
|
|||
enable = true;
|
||||
package = deployCfg.victorialogs.package;
|
||||
|
||||
# ⚠️ PINNED TO LOOPBACK: upstream's default listens on every
|
||||
# interface, and this endpoint accepts writes as well as reads.
|
||||
# Nothing authenticates it either — upstream offers one basic-auth
|
||||
# pair (`-httpAuth.username` / `-httpAuth.password`) and this
|
||||
# module sets neither. The bind address is the boundary.
|
||||
listenAddress = "127.0.0.1:${toString deployCfg.victorialogs.port}";
|
||||
# ⚠️ A SOCKET, not upstream's default of every interface: this
|
||||
# endpoint accepts writes as well as reads. Nothing authenticates it
|
||||
# either — upstream offers one basic-auth pair
|
||||
# (`-httpAuth.username` / `-httpAuth.password`) and this module sets
|
||||
# neither. The socket's directory mode is the boundary: only the
|
||||
# gateway's group can connect.
|
||||
listenAddress = "unix:${socketPath}";
|
||||
|
||||
extraOptions = [ "-retentionPeriod=${deployCfg.victorialogs.retentionPeriod}" ];
|
||||
};
|
||||
|
|
@ -749,7 +780,7 @@ in
|
|||
# code is not evidence.
|
||||
#
|
||||
# ⚠️ Like the metrics store's, that endpoint is unauthenticated — which is
|
||||
# why `listenAddress` above is loopback and why nothing reaches it except
|
||||
# why `listenAddress` above is a socket and why nothing reaches it except
|
||||
# through the gateway, where the ingest route is authenticated. (This module
|
||||
# declares a vhost.)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue