swarm-victorialogs: own network namespace, gateway over a unix socket
This commit is contained in:
parent
c21b158884
commit
188dbcea8b
6 changed files with 155 additions and 76 deletions
|
|
@ -188,10 +188,12 @@ swarm.
|
||||||
| `deploy.victorialogs.authVerdictTtl` | Default `60` seconds that `swarm-logs-auth` keeps granting an agent secret it verified, without asking the secret store. A replaced or deleted secret keeps working at the log store for up to that long. A revoked `logs` permission does too. |
|
| `deploy.victorialogs.authVerdictTtl` | Default `60` seconds that `swarm-logs-auth` keeps granting an agent secret it verified, without asking the secret store. A replaced or deleted secret keeps working at the log store for up to that long. A revoked `logs` permission does too. |
|
||||||
| `deploy.victorialogs.grafanaAuthPackage` | The `swarm-metrics-auth` build that runs as `swarm-logs-grafana-auth`. Set on its own: `deploy.victoriametrics.authPackage` doesn't reach it. |
|
| `deploy.victorialogs.grafanaAuthPackage` | The `swarm-metrics-auth` build that runs as `swarm-logs-grafana-auth`. Set on its own: `deploy.victoriametrics.authPackage` doesn't reach it. |
|
||||||
| `swarm.victorialogs.domain` | Only to rename it. |
|
| `swarm.victorialogs.domain` | Only to rename it. |
|
||||||
| `deploy.victorialogs.port` | Only if something else on the services host already claims `9428`. |
|
|
||||||
|
|
||||||
It binds loopback and takes no credential of its own: the gateway vhost is
|
It listens on a unix socket in a network namespace of its own, on the hive
|
||||||
the only way in, and the collector is the only intended writer.
|
bridge so its journal forwarder can reach the swarm collector, and binds no host
|
||||||
|
port. Only the gateway's nginx can open the socket. It takes no credential of
|
||||||
|
its own: the gateway vhost is the only way in, and the collector is the only
|
||||||
|
intended writer.
|
||||||
|
|
||||||
**Who may query.** `/select/logsql/` takes one of three credentials. An
|
**Who may query.** `/select/logsql/` takes one of three credentials. An
|
||||||
agent presents its `swarm-agent.<agent>.<secret>` bearer, checked by
|
agent presents its `swarm-agent.<agent>.<secret>` bearer, checked by
|
||||||
|
|
|
||||||
|
|
@ -85,6 +85,16 @@ let
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
imports = [
|
imports = [
|
||||||
|
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "victorialogs" "port" ] ''
|
||||||
|
VictoriaLogs listens on a unix socket in a network namespace of its own,
|
||||||
|
and the gateway reaches it there, so it binds no host port. Remove this
|
||||||
|
definition.
|
||||||
|
'')
|
||||||
|
(lib.mkRemovedOptionModule [ "services" "hyperhive" "deploy" "victorialogs" "port" ] ''
|
||||||
|
VictoriaLogs listens on a unix socket in a network namespace of its own,
|
||||||
|
and the gateway reaches it there, so it binds no host port. Remove this
|
||||||
|
definition.
|
||||||
|
'')
|
||||||
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "victoriametrics" "port" ] ''
|
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "victoriametrics" "port" ] ''
|
||||||
VictoriaMetrics listens on a unix socket in a network namespace of its
|
VictoriaMetrics listens on a unix socket in a network namespace of its
|
||||||
own, and the gateway reaches it there, so it binds no host port. Remove
|
own, and the gateway reaches it there, so it binds no host port. Remove
|
||||||
|
|
@ -517,10 +527,6 @@ in
|
||||||
[ "services" "hyperhive" "swarm" "otel" "telemetryPort" ]
|
[ "services" "hyperhive" "swarm" "otel" "telemetryPort" ]
|
||||||
[ "services" "hyperhive" "deploy" "swarm-otel" "telemetryPort" ]
|
[ "services" "hyperhive" "deploy" "swarm-otel" "telemetryPort" ]
|
||||||
)
|
)
|
||||||
(lib.mkRenamedOptionModule
|
|
||||||
[ "services" "hyperhive" "swarm" "victorialogs" "port" ]
|
|
||||||
[ "services" "hyperhive" "deploy" "victorialogs" "port" ]
|
|
||||||
)
|
|
||||||
(lib.mkRenamedOptionModule
|
(lib.mkRenamedOptionModule
|
||||||
[ "services" "hyperhive" "swarm" "forge" "httpPort" ]
|
[ "services" "hyperhive" "swarm" "forge" "httpPort" ]
|
||||||
[ "services" "hyperhive" "deploy" "forgejo" "httpPort" ]
|
[ "services" "hyperhive" "deploy" "forgejo" "httpPort" ]
|
||||||
|
|
|
||||||
|
|
@ -17,8 +17,8 @@
|
||||||
# below, same shape as `swarm-ui.nix`'s) — see that file's copy for the full
|
# below, same shape as `swarm-ui.nix`'s) — see that file's copy for the full
|
||||||
# rationale (forceSSL is load-bearing there too: authelia answers a plain-http
|
# rationale (forceSSL is load-bearing there too: authelia answers a plain-http
|
||||||
# auth subrequest with 400, which `auth_request` cannot interpret as anything
|
# auth subrequest with 400, which `auth_request` cannot interpret as anything
|
||||||
# but a broken check). The store's own listener stays loopback-only and
|
# but a broken check). The store's own listener is a unix socket in a
|
||||||
# unauthenticated, and the vhost is now the ONLY way in from outside: the
|
# netns of its own and is unauthenticated, and the vhost is the ONLY way in: the
|
||||||
# collector pushes through it too, at its own `= /insert/...` location, because
|
# collector pushes through it too, at its own `= /insert/...` location, because
|
||||||
# a swarm has one log store and the collector need not share a host with it.
|
# a swarm has one log store and the collector need not share a host with it.
|
||||||
# ⚠️ That ingest location deliberately does not carry `swarmAuthRequest` — a
|
# ⚠️ That ingest location deliberately does not carry `swarmAuthRequest` — a
|
||||||
|
|
@ -100,9 +100,23 @@ let
|
||||||
grafanaAuthCredential = id: "/run/credentials/${grafanaAuthUnit}.service/${id}";
|
grafanaAuthCredential = id: "/run/credentials/${grafanaAuthUnit}.service/${id}";
|
||||||
grafanaBearerStorePath = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer";
|
grafanaBearerStorePath = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer";
|
||||||
|
|
||||||
# Shared host netns, like every sibling swarm container: Grafana reaches
|
# A netns of its own on the hive bridge, as the cache's container has
|
||||||
# this at 127.0.0.1:<port>.
|
# (./swarm-attic.nix): the forwarder below dials the swarm collector, authelia
|
||||||
privateNetwork = false;
|
# and the store by name, so the container keeps an interface and the bridge's
|
||||||
|
# resolver. The store listens on a unix socket only, so no host port is
|
||||||
|
# bound. Grafana reaches the store by its domain through the gateway.
|
||||||
|
privateNetwork = true;
|
||||||
|
|
||||||
|
# The unix socket the gateway's nginx connects to. The directory's mode is
|
||||||
|
# what keeps it private, so it holds this one socket.
|
||||||
|
socketDir = "/run/swarm-victorialogs";
|
||||||
|
socketPath = "${socketDir}/victorialogs.sock";
|
||||||
|
socketGroup = "gateway-nginx";
|
||||||
|
# Only the number crosses into the container, which knows the group by name.
|
||||||
|
nginxGid = config.ids.gids.nginx;
|
||||||
|
# The upstream every location below forwards to. A location that adds a path
|
||||||
|
# writes `<upstream>:<path>`; one with no path leaves the request's own.
|
||||||
|
upstream = "http://unix:${socketPath}";
|
||||||
|
|
||||||
# The container's journal forwarder pushes to its own receiver on the swarm
|
# The container's journal forwarder pushes to its own receiver on the swarm
|
||||||
# collector, the route `serviceProducers.swarm-victorialogs` in
|
# collector, the route `serviceProducers.swarm-victorialogs` in
|
||||||
|
|
@ -110,8 +124,7 @@ let
|
||||||
forwarderClientId = cfg.otel.clientId;
|
forwarderClientId = cfg.otel.clientId;
|
||||||
forwarderEndpoint = "https://${hyperhiveCfg.swarm.otel.domain}/${cfg.machine}";
|
forwarderEndpoint = "https://${hyperhiveCfg.swarm.otel.domain}/${cfg.machine}";
|
||||||
forwarderUnit = "swarm-journal-forwarder-oidc";
|
forwarderUnit = "swarm-journal-forwarder-oidc";
|
||||||
# In the host netns: distinct from every other collector's there, which the
|
# Loopback, in the container's own netns.
|
||||||
# assertion below checks.
|
|
||||||
forwarderTelemetryPort = 8897;
|
forwarderTelemetryPort = 8897;
|
||||||
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
|
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
|
||||||
# The responder's store identity, handed into the container as system
|
# The responder's store identity, handed into the container as system
|
||||||
|
|
@ -134,7 +147,7 @@ in
|
||||||
{
|
{
|
||||||
# What the store IS from any hive's point of view, the name it answers on, is
|
# What the store IS from any hive's point of view, the name it answers on, is
|
||||||
# `swarm.victorialogs` in ./swarm-victorialogs-service.nix.
|
# `swarm.victorialogs` in ./swarm-victorialogs-service.nix.
|
||||||
# `enable`, `package`, `port` and `retentionPeriod` are decisions of the host that
|
# `enable`, `package` and `retentionPeriod` are decisions of the host that
|
||||||
# runs it and live under `deploy.*`.
|
# runs it and live under `deploy.*`.
|
||||||
|
|
||||||
# Retention is a property of the store this host runs, not something the
|
# Retention is a property of the store this host runs, not something the
|
||||||
|
|
@ -149,22 +162,6 @@ in
|
||||||
description = "VictoriaLogs package to run.";
|
description = "VictoriaLogs package to run.";
|
||||||
};
|
};
|
||||||
|
|
||||||
options.services.hyperhive.deploy.victorialogs.port = lib.mkOption {
|
|
||||||
type = lib.types.port;
|
|
||||||
default = 9428;
|
|
||||||
description = ''
|
|
||||||
Loopback port the store listens on. Upstream's default, kept
|
|
||||||
because there is no reason to move it and a familiar number is
|
|
||||||
one less thing an operator has to look up.
|
|
||||||
|
|
||||||
⚠️ Every swarm container shares the host's network namespace, so
|
|
||||||
this is a swarm-wide claim rather than a per-container one — two
|
|
||||||
modules picking the same number collide at runtime with no bind
|
|
||||||
error and nothing in any log. No repo-wide
|
|
||||||
check catches it.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
options.services.hyperhive.deploy.victorialogs.retentionPeriod = lib.mkOption {
|
options.services.hyperhive.deploy.victorialogs.retentionPeriod = lib.mkOption {
|
||||||
type = lib.types.str;
|
type = lib.types.str;
|
||||||
default = "30d";
|
default = "30d";
|
||||||
|
|
@ -310,20 +307,6 @@ in
|
||||||
Declare it in swarm-otel-service.nix.
|
Declare it in swarm-otel-service.nix.
|
||||||
'';
|
'';
|
||||||
}
|
}
|
||||||
{
|
|
||||||
assertion =
|
|
||||||
!(lib.elem forwarderTelemetryPort [
|
|
||||||
hyperhiveCfg.otel.telemetryPort
|
|
||||||
hyperhiveCfg.deploy.bao.otel.telemetryPort
|
|
||||||
hyperhiveCfg.deploy.nats.otel.telemetryPort
|
|
||||||
]);
|
|
||||||
message = ''
|
|
||||||
The log store's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
|
|
||||||
collides with another collector's in the host netns: the hive
|
|
||||||
collector's, the store forwarder's or the
|
|
||||||
queue forwarder's.
|
|
||||||
'';
|
|
||||||
}
|
|
||||||
];
|
];
|
||||||
|
|
||||||
# nspawn fails to start the container when a credential source is
|
# nspawn fails to start the container when a credential source is
|
||||||
|
|
@ -375,9 +358,9 @@ in
|
||||||
services.hyperhive.gateway.enable = lib.mkDefault true;
|
services.hyperhive.gateway.enable = lib.mkDefault true;
|
||||||
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
||||||
|
|
||||||
# Authenticated front door onto the loopback-only store — see the
|
# Authenticated front door onto the store's socket — see the
|
||||||
# file-top comment for why this is safe to add without touching the
|
# file-top comment for why this is safe to add without touching the
|
||||||
# store's own (still unauthenticated, still loopback) listener at all.
|
# store's own (still unauthenticated, socket-only) listener at all.
|
||||||
# `forceSSL`: same asymmetry `swarm-ui.nix` documents —
|
# `forceSSL`: same asymmetry `swarm-ui.nix` documents —
|
||||||
# authelia answers a plain-http auth subrequest with 400, which
|
# authelia answers a plain-http auth subrequest with 400, which
|
||||||
# `auth_request` cannot read as anything but a broken check, so this
|
# `auth_request` cannot read as anything but a broken check, so this
|
||||||
|
|
@ -387,7 +370,7 @@ in
|
||||||
forceSSL = true;
|
forceSSL = true;
|
||||||
locations = {
|
locations = {
|
||||||
"/" = {
|
"/" = {
|
||||||
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/";
|
proxyPass = "${upstream}:/";
|
||||||
extraConfig = swarmAuthRequest;
|
extraConfig = swarmAuthRequest;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -406,7 +389,7 @@ in
|
||||||
# note on `swarmAuthRequest` above), so naming it here is required
|
# note on `swarmAuthRequest` above), so naming it here is required
|
||||||
# rather than redundant.
|
# rather than redundant.
|
||||||
"= /insert/opentelemetry/v1/logs" = {
|
"= /insert/opentelemetry/v1/logs" = {
|
||||||
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/insert/opentelemetry/v1/logs";
|
proxyPass = "${upstream}:/insert/opentelemetry/v1/logs";
|
||||||
extraConfig = ''
|
extraConfig = ''
|
||||||
auth_request /__swarm_logs_ingest_authz;
|
auth_request /__swarm_logs_ingest_authz;
|
||||||
'';
|
'';
|
||||||
|
|
@ -440,7 +423,7 @@ in
|
||||||
"^~ /select/logsql/" = {
|
"^~ /select/logsql/" = {
|
||||||
# No URI part, so the request path and its query string reach
|
# No URI part, so the request path and its query string reach
|
||||||
# the store as the caller sent them.
|
# the store as the caller sent them.
|
||||||
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}";
|
proxyPass = upstream;
|
||||||
extraConfig = ''
|
extraConfig = ''
|
||||||
auth_request /__swarm_logs_auth;
|
auth_request /__swarm_logs_auth;
|
||||||
'';
|
'';
|
||||||
|
|
@ -457,7 +440,7 @@ in
|
||||||
# authelia's rules for this URL admit the collector's client and
|
# authelia's rules for this URL admit the collector's client and
|
||||||
# deny every other subject (see `metricsRules` in ./swarm-authelia.nix).
|
# deny every other subject (see `metricsRules` in ./swarm-authelia.nix).
|
||||||
"= /metrics" = {
|
"= /metrics" = {
|
||||||
proxyPass = "http://127.0.0.1:${toString deployCfg.victorialogs.port}/metrics";
|
proxyPass = "${upstream}:/metrics";
|
||||||
extraConfig = ''
|
extraConfig = ''
|
||||||
auth_request /__hive_authelia;
|
auth_request /__hive_authelia;
|
||||||
'';
|
'';
|
||||||
|
|
@ -659,6 +642,12 @@ in
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
# The socket directory is the gateway's way in: its group is nginx's.
|
||||||
|
systemd.tmpfiles.rules = [ "d ${socketDir} 0770 root ${toString nginxGid} - -" ];
|
||||||
|
|
||||||
|
# The hive bridge carries the container's DHCP lease and its resolver.
|
||||||
|
services.hyperhive.network.enable = lib.mkDefault true;
|
||||||
|
|
||||||
containers.${cfg.machine} = {
|
containers.${cfg.machine} = {
|
||||||
autoStart = true;
|
autoStart = true;
|
||||||
ephemeral = false;
|
ephemeral = false;
|
||||||
|
|
@ -668,10 +657,20 @@ in
|
||||||
id: source: "--load-credential=${id}:${source}"
|
id: source: "--load-credential=${id}:${source}"
|
||||||
) forwarderStoreFiles;
|
) forwarderStoreFiles;
|
||||||
inherit privateNetwork;
|
inherit privateNetwork;
|
||||||
bindMounts = caTrust.bindMount;
|
hostBridge = networkCfg.bridgeName;
|
||||||
|
# A bind mount, not a path both sides know: a container's `/run` is an
|
||||||
|
# nspawn tmpfs the host cannot see into. Connecting to a socket is a
|
||||||
|
# write to it.
|
||||||
|
bindMounts = {
|
||||||
|
${socketDir} = {
|
||||||
|
hostPath = socketDir;
|
||||||
|
isReadOnly = false;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
// caTrust.bindMount;
|
||||||
|
|
||||||
config =
|
config =
|
||||||
{ ... }:
|
{ pkgs, ... }:
|
||||||
{
|
{
|
||||||
imports = [
|
imports = [
|
||||||
../container-modules/swarm-container.nix
|
../container-modules/swarm-container.nix
|
||||||
|
|
@ -697,6 +696,37 @@ in
|
||||||
|
|
||||||
services.hyperhive.swarmContainer = { inherit privateNetwork; };
|
services.hyperhive.swarmContainer = { inherit privateNetwork; };
|
||||||
|
|
||||||
|
# The bridge address comes from the gateway's DHCP. `swarm-bridge-dns`
|
||||||
|
# owns resolv.conf, so dhcpcd must not write it over that unit.
|
||||||
|
networking.interfaces.eth0.useDHCP = true;
|
||||||
|
networking.dhcpcd.extraConfig = "nohook resolv.conf";
|
||||||
|
|
||||||
|
# The socket's group is the store's primary group, so that group is
|
||||||
|
# nginx's gid. A static user, not nixpkgs' DynamicUser: a dynamic
|
||||||
|
# user's group is allocated per start, and nginx is never in it.
|
||||||
|
users.groups.${socketGroup}.gid = nginxGid;
|
||||||
|
users.users.victorialogs = {
|
||||||
|
isSystemUser = true;
|
||||||
|
group = socketGroup;
|
||||||
|
};
|
||||||
|
systemd.services.victorialogs = {
|
||||||
|
serviceConfig = {
|
||||||
|
DynamicUser = lib.mkForce false;
|
||||||
|
User = "victorialogs";
|
||||||
|
Group = socketGroup;
|
||||||
|
# The store creates its socket under the process umask, so `0007`
|
||||||
|
# makes it `0770` and nginx's group can connect.
|
||||||
|
UMask = "0007";
|
||||||
|
};
|
||||||
|
# nixpkgs' readiness loop curls a TCP URL, which no `unix:` address
|
||||||
|
# answers.
|
||||||
|
postStart = lib.mkForce ''
|
||||||
|
until ${lib.getExe pkgs.curl} -s -o /dev/null --unix-socket ${socketPath} http://localhost/ping; do
|
||||||
|
sleep 1;
|
||||||
|
done
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
# Authenticated as the forwarder's own OIDC client, with the
|
# Authenticated as the forwarder's own OIDC client, with the
|
||||||
# responder's store identity: one identity for the whole container.
|
# responder's store identity: one identity for the whole container.
|
||||||
# It is a system credential, so the fetch unit loads it.
|
# It is a system credential, so the fetch unit loads it.
|
||||||
|
|
@ -722,12 +752,13 @@ in
|
||||||
enable = true;
|
enable = true;
|
||||||
package = deployCfg.victorialogs.package;
|
package = deployCfg.victorialogs.package;
|
||||||
|
|
||||||
# ⚠️ PINNED TO LOOPBACK: upstream's default listens on every
|
# ⚠️ A SOCKET, not upstream's default of every interface: this
|
||||||
# interface, and this endpoint accepts writes as well as reads.
|
# endpoint accepts writes as well as reads. Nothing authenticates it
|
||||||
# Nothing authenticates it either — upstream offers one basic-auth
|
# either — upstream offers one basic-auth pair
|
||||||
# pair (`-httpAuth.username` / `-httpAuth.password`) and this
|
# (`-httpAuth.username` / `-httpAuth.password`) and this module sets
|
||||||
# module sets neither. The bind address is the boundary.
|
# neither. The socket's directory mode is the boundary: only the
|
||||||
listenAddress = "127.0.0.1:${toString deployCfg.victorialogs.port}";
|
# gateway's group can connect.
|
||||||
|
listenAddress = "unix:${socketPath}";
|
||||||
|
|
||||||
extraOptions = [ "-retentionPeriod=${deployCfg.victorialogs.retentionPeriod}" ];
|
extraOptions = [ "-retentionPeriod=${deployCfg.victorialogs.retentionPeriod}" ];
|
||||||
};
|
};
|
||||||
|
|
@ -749,7 +780,7 @@ in
|
||||||
# code is not evidence.
|
# code is not evidence.
|
||||||
#
|
#
|
||||||
# ⚠️ Like the metrics store's, that endpoint is unauthenticated — which is
|
# ⚠️ Like the metrics store's, that endpoint is unauthenticated — which is
|
||||||
# why `listenAddress` above is loopback and why nothing reaches it except
|
# why `listenAddress` above is a socket and why nothing reaches it except
|
||||||
# through the gateway, where the ingest route is authenticated. (This module
|
# through the gateway, where the ingest route is authenticated. (This module
|
||||||
# declares a vhost.)
|
# declares a vhost.)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -2262,10 +2262,10 @@ let
|
||||||
{
|
{
|
||||||
# Each old spelling lands on the new path, with its own rename warning,
|
# Each old spelling lands on the new path, with its own rename warning,
|
||||||
# and its value reaches the unit or vhost that reads it. Ports are chosen
|
# and its value reaches the unit or vhost that reads it. Ports are chosen
|
||||||
# so three of them collide on purpose: the bao collector's telemetry
|
# so two of them collide on purpose: the bao collector's telemetry
|
||||||
# port equals the NATS monitor port and the forge forwarder's, and the
|
# port equals the NATS monitor port and the forge forwarder's. Each of the
|
||||||
# NATS forwarder's equals the log store forwarder's. Each of the three
|
# two colocation assertions must read the moved options to see it. (The log
|
||||||
# colocation assertions must read the moved options to see it.
|
# store has a netns of its own, so its forwarder's port collides with none.)
|
||||||
name = "the per-service settings moved to deploy.* still apply from their old paths, with a rename warning each";
|
name = "the per-service settings moved to deploy.* still apply from their old paths, with a rename warning each";
|
||||||
ok =
|
ok =
|
||||||
let
|
let
|
||||||
|
|
@ -2285,7 +2285,6 @@ let
|
||||||
nats.metricsPort = 17777;
|
nats.metricsPort = 17777;
|
||||||
nats.otel.telemetryPort = 8897;
|
nats.otel.telemetryPort = 8897;
|
||||||
otel.telemetryPort = 18889;
|
otel.telemetryPort = 18889;
|
||||||
victorialogs.port = 19428;
|
|
||||||
forge.httpPort = 13000;
|
forge.httpPort = 13000;
|
||||||
matrix.allowEncryption = true;
|
matrix.allowEncryption = true;
|
||||||
controller.links = [
|
controller.links = [
|
||||||
|
|
@ -2322,7 +2321,6 @@ let
|
||||||
&& d.nats.metricsPort == 17777
|
&& d.nats.metricsPort == 17777
|
||||||
&& d.nats.otel.telemetryPort == 8897
|
&& d.nats.otel.telemetryPort == 8897
|
||||||
&& d.swarm-otel.telemetryPort == 18889
|
&& d.swarm-otel.telemetryPort == 18889
|
||||||
&& d.victorialogs.port == 19428
|
|
||||||
&& d.forgejo.httpPort == 13000
|
&& d.forgejo.httpPort == 13000
|
||||||
&& d.matrix.allowEncryption
|
&& d.matrix.allowEncryption
|
||||||
&& d.builder.nixpkgsFlake == "path:/legacy/nixpkgs"
|
&& d.builder.nixpkgsFlake == "path:/legacy/nixpkgs"
|
||||||
|
|
@ -2343,7 +2341,6 @@ let
|
||||||
old.containers.swarm-builder.config.systemd.services.swarm-builder.environment.SWARM_BUILDER_NIXPKGS_FLAKE
|
old.containers.swarm-builder.config.systemd.services.swarm-builder.environment.SWARM_BUILDER_NIXPKGS_FLAKE
|
||||||
== "path:/legacy/nixpkgs"
|
== "path:/legacy/nixpkgs"
|
||||||
&& (locs sw.nats.domain)."= /metrics".proxyPass == "http://127.0.0.1:17777/metrics"
|
&& (locs sw.nats.domain)."= /metrics".proxyPass == "http://127.0.0.1:17777/metrics"
|
||||||
&& (locs sw.victorialogs.domain)."= /metrics".proxyPass == "http://127.0.0.1:19428/metrics"
|
|
||||||
&& (locs sw.forge.domain)."/".proxyPass == "http://127.0.0.1:13000/"
|
&& (locs sw.forge.domain)."/".proxyPass == "http://127.0.0.1:13000/"
|
||||||
&& old.containers.hive-matrix.config.services.matrix-tuwunel.settings.global.allow_encryption
|
&& old.containers.hive-matrix.config.services.matrix-tuwunel.settings.global.allow_encryption
|
||||||
&& lib.all warned [
|
&& lib.all warned [
|
||||||
|
|
@ -2354,7 +2351,6 @@ let
|
||||||
"swarm.nats.metricsPort"
|
"swarm.nats.metricsPort"
|
||||||
"swarm.nats.otel.telemetryPort"
|
"swarm.nats.otel.telemetryPort"
|
||||||
"swarm.otel.telemetryPort"
|
"swarm.otel.telemetryPort"
|
||||||
"swarm.victorialogs.port"
|
|
||||||
"swarm.forge.httpPort"
|
"swarm.forge.httpPort"
|
||||||
"swarm.matrix.allowEncryption"
|
"swarm.matrix.allowEncryption"
|
||||||
"c0re.nixpkgsFlake"
|
"c0re.nixpkgsFlake"
|
||||||
|
|
@ -2365,10 +2361,8 @@ let
|
||||||
&& !(lib.any (w: lib.hasInfix "swarm.controller.queue" w) allLocal.warnings)
|
&& !(lib.any (w: lib.hasInfix "swarm.controller.queue" w) allLocal.warnings)
|
||||||
&& refused old "deploy.nats.otel.telemetryPort"
|
&& refused old "deploy.nats.otel.telemetryPort"
|
||||||
&& refused old "The forge's journal forwarder's telemetry port"
|
&& refused old "The forge's journal forwarder's telemetry port"
|
||||||
&& refused old "The log store's journal forwarder's telemetry port"
|
|
||||||
&& !(refused allLocal "deploy.nats.otel.telemetryPort")
|
&& !(refused allLocal "deploy.nats.otel.telemetryPort")
|
||||||
&& !(refused allLocal "The forge's journal forwarder's telemetry port")
|
&& !(refused allLocal "The forge's journal forwarder's telemetry port");
|
||||||
&& !(refused allLocal "The log store's journal forwarder's telemetry port");
|
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
# The default render of the moved options, pinned so moving them changes
|
# The default render of the moved options, pinned so moving them changes
|
||||||
|
|
@ -2389,7 +2383,9 @@ let
|
||||||
&& h.deploy.nats.otel.telemetryPort == 8891
|
&& h.deploy.nats.otel.telemetryPort == 8891
|
||||||
&& h.deploy.swarm-otel.telemetryPort == 8889
|
&& h.deploy.swarm-otel.telemetryPort == 8889
|
||||||
&& (locs h.swarm.nats.domain)."= /metrics".proxyPass == "http://127.0.0.1:7777/metrics"
|
&& (locs h.swarm.nats.domain)."= /metrics".proxyPass == "http://127.0.0.1:7777/metrics"
|
||||||
&& (locs h.swarm.victorialogs.domain)."= /metrics".proxyPass == "http://127.0.0.1:9428/metrics"
|
&&
|
||||||
|
(locs h.swarm.victorialogs.domain)."= /metrics".proxyPass
|
||||||
|
== "http://unix:/run/swarm-victorialogs/victorialogs.sock:/metrics"
|
||||||
&& (locs h.swarm.forge.domain)."/".proxyPass == "http://127.0.0.1:3000/"
|
&& (locs h.swarm.forge.domain)."/".proxyPass == "http://127.0.0.1:3000/"
|
||||||
&& !allLocal.containers.hive-matrix.config.services.matrix-tuwunel.settings.global.allow_encryption
|
&& !allLocal.containers.hive-matrix.config.services.matrix-tuwunel.settings.global.allow_encryption
|
||||||
&& env.SWARM_CONTROLLER_MATRIX_ADMIN_SYNC_IGNORE_USERS == ""
|
&& env.SWARM_CONTROLLER_MATRIX_ADMIN_SYNC_IGNORE_USERS == ""
|
||||||
|
|
|
||||||
|
|
@ -511,7 +511,7 @@ let
|
||||||
in
|
in
|
||||||
h.deploy.victorialogs.enable
|
h.deploy.victorialogs.enable
|
||||||
&& locs ? "= /metrics"
|
&& locs ? "= /metrics"
|
||||||
&& locs."= /metrics".proxyPass == "http://127.0.0.1:${toString h.deploy.victorialogs.port}/metrics"
|
&& locs."= /metrics".proxyPass == "http://unix:/run/swarm-victorialogs/victorialogs.sock:/metrics"
|
||||||
&& checksOf locs."= /metrics" == [ "auth_request /__hive_authelia;" ]
|
&& checksOf locs."= /metrics" == [ "auth_request /__hive_authelia;" ]
|
||||||
&& !(lib.hasInfix "error_page" locs."= /metrics".extraConfig)
|
&& !(lib.hasInfix "error_page" locs."= /metrics".extraConfig)
|
||||||
&& lib.hasInfix "error_page 401 =302" locs."/".extraConfig
|
&& lib.hasInfix "error_page 401 =302" locs."/".extraConfig
|
||||||
|
|
|
||||||
|
|
@ -843,14 +843,14 @@ let
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
# Both values come from ../container-modules/swarm-container.nix.
|
# Both values come from ../container-modules/swarm-container.nix.
|
||||||
# Four of the 16 service containers share the host netns (hive-forge, bao,
|
# Three of the 16 service containers share the host netns (hive-forge, bao,
|
||||||
# the queue, the log store) and twelve have one of their own.
|
# the queue) and thirteen have one of their own.
|
||||||
name = "a service container on the host netns runs no firewall or resolvconf of its own";
|
name = "a service container on the host netns runs no firewall or resolvconf of its own";
|
||||||
ok =
|
ok =
|
||||||
let
|
let
|
||||||
shared = lib.filter (c: !c.privateNetwork) (lib.attrValues serviceContainers);
|
shared = lib.filter (c: !c.privateNetwork) (lib.attrValues serviceContainers);
|
||||||
in
|
in
|
||||||
lib.length shared == 4
|
lib.length shared == 3
|
||||||
&& lib.all (
|
&& lib.all (
|
||||||
c: !c.config.networking.firewall.enable && !c.config.networking.resolvconf.enable
|
c: !c.config.networking.firewall.enable && !c.config.networking.resolvconf.enable
|
||||||
) shared;
|
) shared;
|
||||||
|
|
@ -990,6 +990,50 @@ let
|
||||||
&& bridge.serviceConfig.Group == "gateway-nginx"
|
&& bridge.serviceConfig.Group == "gateway-nginx"
|
||||||
&& lib.elem dir unit.serviceConfig.ReadWritePaths;
|
&& lib.elem dir unit.serviceConfig.ReadWritePaths;
|
||||||
}
|
}
|
||||||
|
{
|
||||||
|
# On the bridge by DHCP, listening on a socket only the gateway's group can
|
||||||
|
# reach, so the store binds no host port. Every location that forwards to
|
||||||
|
# it goes through the socket (the authelia subrequests are the `https://`
|
||||||
|
# upstream); the read, ingest, scrape and catch-all locations are named so
|
||||||
|
# an empty filter can't pass, and the old loopback port is named nowhere.
|
||||||
|
name = "the log store runs on a netns of its own and the gateway reaches it over its socket";
|
||||||
|
ok =
|
||||||
|
let
|
||||||
|
host = serviceContainersWithCi;
|
||||||
|
net = host.services.hyperhive.network;
|
||||||
|
c = host.containers.swarm-victorialogs;
|
||||||
|
dir = "/run/swarm-victorialogs";
|
||||||
|
sock = "${dir}/victorialogs.sock";
|
||||||
|
viaSock = path: "http://unix:${sock}:${path}";
|
||||||
|
locs =
|
||||||
|
host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.victorialogs.domain}.locations;
|
||||||
|
toStore = lib.filter (
|
||||||
|
p: p != null && !(lib.hasPrefix "https://" p) && !(lib.hasPrefix "http://unix:/run/swarm-logs" p)
|
||||||
|
) (map (l: l.proxyPass) (lib.attrValues locs));
|
||||||
|
unit = c.config.systemd.services.victorialogs;
|
||||||
|
nginxGid = host.ids.gids.nginx;
|
||||||
|
in
|
||||||
|
c.privateNetwork
|
||||||
|
&& c.hostBridge == net.bridgeName
|
||||||
|
&& c.hostAddress == null
|
||||||
|
&& c.localAddress == null
|
||||||
|
&& c.config.networking.interfaces.eth0.useDHCP
|
||||||
|
&& c.config.services.victorialogs.listenAddress == "unix:${sock}"
|
||||||
|
&& locs."/".proxyPass == viaSock "/"
|
||||||
|
&& locs."= /insert/opentelemetry/v1/logs".proxyPass == viaSock "/insert/opentelemetry/v1/logs"
|
||||||
|
&& locs."^~ /select/logsql/".proxyPass == "http://unix:${sock}"
|
||||||
|
&& locs."= /metrics".proxyPass == viaSock "/metrics"
|
||||||
|
&& toStore != [ ]
|
||||||
|
&& lib.all (lib.hasPrefix "http://unix:${sock}") toStore
|
||||||
|
&& !(lib.any (lib.hasInfix "127.0.0.1") toStore)
|
||||||
|
&& c.bindMounts.${dir}.hostPath == dir
|
||||||
|
&& builtins.elem "d ${dir} 0770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules
|
||||||
|
&& c.config.users.groups.gateway-nginx.gid == nginxGid
|
||||||
|
&& unit.serviceConfig.Group == "gateway-nginx"
|
||||||
|
&& !unit.serviceConfig.DynamicUser
|
||||||
|
&& unit.serviceConfig.UMask == "0007"
|
||||||
|
&& lib.hasInfix "--unix-socket ${sock}" unit.postStart;
|
||||||
|
}
|
||||||
{
|
{
|
||||||
# On the bridge by DHCP, listening on a socket (protocol `socket` ignores the
|
# On the bridge by DHCP, listening on a socket (protocol `socket` ignores the
|
||||||
# upstream-default `http_port`, which stays in the settings): the
|
# upstream-default `http_port`, which stays in the settings): the
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue