683 lines
30 KiB
Nix
683 lines
30 KiB
Nix
# The swarm's log store: one VictoriaLogs for the whole swarm, in a
|
||
# `swarm-victorialogs` nixos-container beside the metrics store it mirrors.
|
||
#
|
||
# Why a store at all rather than reading journals directly: an agent can
|
||
# verify that a unit was *launched* and never that it is *working*. The
|
||
# container journals are not reachable from an agent, so a diagnosis stops at
|
||
# the first component that is broken — which is precisely the component whose
|
||
# own instrument is least likely to be legible. Collecting logs centrally
|
||
# makes the question answerable without host access.
|
||
#
|
||
# It is the collector that feeds this, not the services directly: one ingest
|
||
# point per swarm, same shape as the metrics path.
|
||
#
|
||
# Gateway vhost is authenticated, unlike the metrics store's: an operator
|
||
# reaches this at `https://${cfg.domain}/` gated by the same `auth_request`
|
||
# check against authelia that swarm-ui's own vhost uses (`swarmAuthRequest`
|
||
# below, same shape as `swarm-ui.nix`'s) — see that file's copy for the full
|
||
# rationale (forceSSL is load-bearing there too: authelia answers a plain-http
|
||
# auth subrequest with 400, which `auth_request` cannot interpret as anything
|
||
# but a broken check). The store's own listener stays loopback-only and
|
||
# unauthenticated, and the vhost is now the ONLY way in from outside: the
|
||
# collector pushes through it too, at its own `= /insert/...` location, because
|
||
# a swarm has one log store and the collector need not share a host with it.
|
||
# ⚠️ That ingest location deliberately does not carry `swarmAuthRequest` — a
|
||
# pusher handed its `error_page 401 =302` follows the redirect and POSTs at a
|
||
# login page, which answers 200. See the location itself.
|
||
#
|
||
# The query API has a machine route of the same shape at `^~ /select/logsql/`,
|
||
# for the same reason, and with no filtering of what an authenticated caller
|
||
# may read — see that location.
|
||
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
cfg = config.services.hyperhive.swarm.victorialogs;
|
||
deployCfg = config.services.hyperhive.deploy;
|
||
networkCfg = config.services.hyperhive.network;
|
||
hyperhiveCfg = config.services.hyperhive;
|
||
gatewayCfg = hyperhiveCfg.gateway;
|
||
|
||
# Copy of `swarm-ui.nix`'s own `swarmAuthRequest` — not shared code because
|
||
# this vhost needs exactly the two locations that reference it (`/` and the
|
||
# internal auth-request target below) and pulling in swarm-ui.nix for one
|
||
# string would couple this module to swarm-ui existing at all, which it
|
||
# need not. Same three lines, same reasoning as that file's own comment.
|
||
swarmAuthRequest = ''
|
||
auth_request /__hive_authelia;
|
||
auth_request_set $target_url $scheme://$http_host$request_uri;
|
||
error_page 401 =302 https://${hyperhiveCfg.swarm.authelia.domain}/?rd=$target_url;
|
||
'';
|
||
|
||
# The agents' `auth_request` responder (`swarm-logs-auth`). Its socket's
|
||
# directory holds that socket alone and is traverse-only for others, the
|
||
# shape `swarm-controller.nix` gives its own.
|
||
logsAuthUnit = "swarm-logs-auth";
|
||
logsAuthSocket = "/run/${logsAuthUnit}/auth.sock";
|
||
logsAuthCredential = id: "/run/credentials/${logsAuthUnit}.service/${id}";
|
||
# Credential id → host path of the responder's store identity.
|
||
logsAuthStoreFiles = {
|
||
"bao-client.pem" = deployCfg.victorialogs.authBaoClientCertFile;
|
||
"bao-client-key.pem" = deployCfg.victorialogs.authBaoClientKeyFile;
|
||
}
|
||
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
|
||
"bao-ca.pem" = deployCfg.bao.serverCaFile;
|
||
};
|
||
baoCfg = hyperhiveCfg.swarm.bao;
|
||
|
||
# Grafana's bearer, checked by a second `swarm-metrics-auth` instance. It
|
||
# reads the store with the same leaf as `swarm-logs-auth`, which is granted
|
||
# the one extra path below. The same leaf as `grafanaBearerStorePath` in
|
||
# ./swarm-victoriametrics.nix and `grafanaMetricsBearerLeaf` in ./swarm-bao.nix.
|
||
grafanaAuthUnit = "swarm-logs-grafana-auth";
|
||
grafanaAuthSocket = "/run/${grafanaAuthUnit}/auth.sock";
|
||
grafanaAuthCredential = id: "/run/credentials/${grafanaAuthUnit}.service/${id}";
|
||
grafanaBearerStorePath = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer";
|
||
|
||
# Shared host netns, like every sibling swarm container: Grafana reaches
|
||
# this at 127.0.0.1:<port>.
|
||
privateNetwork = false;
|
||
|
||
# The container's journal forwarder pushes to its own receiver on the swarm
|
||
# collector, the route `serviceProducers.swarm-victorialogs` in
|
||
# ./swarm-otel-service.nix declares. The port suffix is elided on 443, as in
|
||
# ./swarm-bao.nix.
|
||
forwarderClientId = cfg.otel.clientId;
|
||
gatewayPortSuffix = lib.optionalString (
|
||
gatewayCfg.httpsPort != 443
|
||
) ":${toString gatewayCfg.httpsPort}";
|
||
forwarderEndpoint = "https://${hyperhiveCfg.swarm.otel.domain}${gatewayPortSuffix}/${cfg.machine}";
|
||
forwarderUnit = "swarm-journal-forwarder-oidc";
|
||
# In the host netns: distinct from every other collector's there, which the
|
||
# assertion below checks.
|
||
forwarderTelemetryPort = 8897;
|
||
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
|
||
# The responder's store identity, handed into the container as system
|
||
# credentials for the forwarder's fetch unit. The host units keep reading
|
||
# the same files.
|
||
forwarderStoreFiles = lib.filterAttrs (_: source: source != null) logsAuthStoreFiles;
|
||
# nixos-container expands `extraFlags` unquoted, so a source path must be
|
||
# one word.
|
||
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
|
||
|
||
# The forwarder reaches the swarm collector and authelia by name over TLS; in
|
||
# self-signed mode their leaves chain to the hive CA.
|
||
caTrust = import ./lib/hive-ca-trust.nix {
|
||
inherit lib gatewayCfg;
|
||
tlsCfg = deployCfg.hive-controller.tls;
|
||
};
|
||
in
|
||
{
|
||
# What the store IS from any hive's point of view, the name it answers on and
|
||
# the port, is `swarm.victorialogs` in ./swarm-victorialogs-service.nix.
|
||
# `enable`, `package` and `retentionPeriod` are decisions of the host that
|
||
# runs it and live under `deploy.*`.
|
||
|
||
# Retention is a property of the store this host runs, not something the
|
||
# swarm has to agree on: it is read only where the container is defined,
|
||
# and a hive that is a *client* of the log store never consults it. That
|
||
# makes it a `deploy.*` value by the same rule as the seal on the secret
|
||
# store — options on the auto-deployed service itself.
|
||
options.services.hyperhive.deploy.victorialogs.package = lib.mkOption {
|
||
type = lib.types.package;
|
||
default = pkgs.victorialogs;
|
||
defaultText = lib.literalExpression "pkgs.victorialogs";
|
||
description = "VictoriaLogs package to run.";
|
||
};
|
||
|
||
options.services.hyperhive.deploy.victorialogs.retentionPeriod = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "30d";
|
||
example = "90d";
|
||
description = ''
|
||
How long log data is kept.
|
||
|
||
Deliberately far shorter than the metrics store's retention: logs
|
||
are orders of magnitude larger per unit of time, and their value
|
||
decays much faster. A log line answers "what happened during that
|
||
incident"; a metric answers "is this worse than last quarter".
|
||
'';
|
||
};
|
||
|
||
options.services.hyperhive.deploy.victorialogs.authPackage = lib.mkOption {
|
||
type = lib.types.package;
|
||
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-logs-auth";
|
||
description = ''
|
||
The `swarm-logs-auth` package: the gateway's `auth_request` responder
|
||
that checks an agent's `swarm-agent.<agent>.<secret>` bearer on
|
||
`/select/logsql/` against the secret store.
|
||
'';
|
||
};
|
||
|
||
options.services.hyperhive.deploy.victorialogs.grafanaAuthPackage = lib.mkOption {
|
||
type = lib.types.package;
|
||
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-metrics-auth";
|
||
description = ''
|
||
The `swarm-metrics-auth` package, run as `swarm-logs-grafana-auth`: the
|
||
gateway's `auth_request` responder that checks the bearer Grafana's
|
||
logs datasource presents on `/select/logsql/` against the secret store.
|
||
|
||
⚠️ Not folded into `authPackage` above: that one is `swarm-logs-auth`,
|
||
a different binary. Same split as
|
||
{option}`services.hyperhive.deploy.nats.authPackage`.
|
||
'';
|
||
};
|
||
|
||
options.services.hyperhive.deploy.victorialogs.authBaoClientCertFile = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/var/lib/swarm-bao-pki/logs-auth.pem";
|
||
description = ''
|
||
Client certificate `swarm-logs-auth` presents to the swarm's secret store
|
||
to read agents' queue secrets. Its subject must be
|
||
{option}`services.hyperhive.deploy.bao.logsAuthCommonName`.
|
||
|
||
Required when `deploy.victorialogs.enable` is set; unset fails at eval.
|
||
./glue-logs-auth-bao-identity.nix points it at the leaf
|
||
./glue-bao-tls.nix mints on a host that runs the store; on any other
|
||
log-store host it is a file the operator places.
|
||
|
||
A path, never a value.
|
||
'';
|
||
};
|
||
|
||
options.services.hyperhive.deploy.victorialogs.authBaoClientKeyFile = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/var/lib/swarm-bao-pki/logs-auth-key.pem";
|
||
description = ''
|
||
Private key for {option}`services.hyperhive.deploy.victorialogs.authBaoClientCertFile`.
|
||
A path, never a value.
|
||
'';
|
||
};
|
||
|
||
options.services.hyperhive.deploy.victorialogs.authVerdictTtl = lib.mkOption {
|
||
type = lib.types.ints.unsigned;
|
||
default = 60;
|
||
description = ''
|
||
Seconds `swarm-logs-auth` grants an agent bearer it has verified again
|
||
without reading the secret store. A denial is never remembered.
|
||
|
||
This is also how long a re-minted or deleted queue secret keeps working
|
||
at the log store's query route after its last successful check. `0`
|
||
checks the store on every query.
|
||
'';
|
||
};
|
||
|
||
options.services.hyperhive.deploy.victorialogs.authRate = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "50r/s";
|
||
example = "30r/m";
|
||
description = ''
|
||
nginx `limit_req` rate, per client address, of `/select/logsql/`
|
||
requests carrying a `swarm-agent.` bearer, which nginx checks with
|
||
`swarm-logs-auth`. Each such request that the responder has not granted
|
||
within {option}`services.hyperhive.deploy.victorialogs.authVerdictTtl`
|
||
costs a secret store login and read, whether the bearer is right or not.
|
||
|
||
Counts every agent query, granted or not, and is keyed by client address,
|
||
so every agent on one host shares one budget. The default covers about a
|
||
hundred agents on a host cold-starting at once, each costing one store
|
||
login and read for its first query. Requests over the rate and its burst
|
||
are answered 403.
|
||
'';
|
||
};
|
||
|
||
options.services.hyperhive.deploy.victorialogs.authBurst = lib.mkOption {
|
||
type = lib.types.ints.positive;
|
||
default = 500;
|
||
description = ''
|
||
Requests a client address may make above
|
||
{option}`services.hyperhive.deploy.victorialogs.authRate` before nginx
|
||
answers 403, served without delay.
|
||
'';
|
||
};
|
||
|
||
config = lib.mkIf deployCfg.victorialogs.enable {
|
||
assertions = [
|
||
{
|
||
assertion =
|
||
deployCfg.victorialogs.authBaoClientCertFile != null
|
||
&& deployCfg.victorialogs.authBaoClientKeyFile != null;
|
||
message = ''
|
||
services.hyperhive.deploy.victorialogs.authBaoClientCertFile and
|
||
authBaoClientKeyFile must both be set.
|
||
|
||
swarm-logs-auth reads agents' queue secrets from the swarm's secret
|
||
store with this certificate. They default to the leaf the store
|
||
mints where services.hyperhive.deploy.bao.enable is set on this
|
||
host; elsewhere, issue a leaf whose CN is
|
||
services.hyperhive.deploy.bao.logsAuthCommonName and name it in
|
||
both options.
|
||
'';
|
||
}
|
||
{
|
||
assertion = lib.all oneWord (lib.attrValues forwarderStoreFiles);
|
||
message = ''
|
||
The log store responder's store identity paths must not contain
|
||
whitespace: nspawn's --load-credential receives them through
|
||
nixos-container's unquoted extra flags.
|
||
'';
|
||
}
|
||
{
|
||
assertion =
|
||
(hyperhiveCfg.swarm.otel.serviceProducers.${cfg.machine}.clientId or null) == forwarderClientId;
|
||
message = ''
|
||
services.hyperhive.swarm.otel.serviceProducers has no entry
|
||
"${cfg.machine}" with clientId "${forwarderClientId}", so the swarm
|
||
collector has no receiver for the log store's journal forwarder.
|
||
Declare it in swarm-otel-service.nix.
|
||
'';
|
||
}
|
||
{
|
||
assertion =
|
||
!(lib.elem forwarderTelemetryPort [
|
||
hyperhiveCfg.otel.telemetryPort
|
||
hyperhiveCfg.swarm.otel.telemetryPort
|
||
hyperhiveCfg.swarm.bao.otel.telemetryPort
|
||
hyperhiveCfg.swarm.nats.otel.telemetryPort
|
||
]);
|
||
message = ''
|
||
The log store's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
|
||
collides with another collector's in the host netns: the hive
|
||
collector's, the swarm collector's, the store forwarder's or the
|
||
queue forwarder's.
|
||
'';
|
||
}
|
||
];
|
||
|
||
# nspawn fails to start the container when a credential source is
|
||
# missing, so where this host mints the leaf the container pulls the
|
||
# minting unit in and waits for it, as ./swarm-builder.nix's does. The CA
|
||
# ordering keeps the trust bundle's bind source there first.
|
||
systemd.services."container@${cfg.machine}" = lib.mkMerge [
|
||
caTrust.containerOrdering
|
||
{
|
||
after = [ "swarm-bao-pki.service" ];
|
||
wants = [ "swarm-bao-pki.service" ];
|
||
}
|
||
];
|
||
|
||
# Which check a `/select/logsql/` request gets. `auth_request` takes a
|
||
# fixed URI, so it names `/__swarm_logs_auth`, which rewrites to the
|
||
# target this map picks from the caller's `Authorization`.
|
||
services.nginx.appendHttpConfig = ''
|
||
map $http_authorization $swarm_logs_auth_target {
|
||
"~^Bearer swarm-agent\." /__swarm_logs_agent_auth;
|
||
"~^Bearer [0-9a-f]{64}$" /__swarm_logs_grafana_auth;
|
||
default /__hive_authelia;
|
||
}
|
||
limit_req_zone $binary_remote_addr zone=swarm_logs_agent_auth:1m rate=${deployCfg.victorialogs.authRate};
|
||
'';
|
||
|
||
# The gateway name, inside `deployCfg.victorialogs.enable` — same
|
||
# "only the host that runs the service may claim the name" guard every
|
||
# sibling swarm-service module uses (`swarm-grafana.nix`,
|
||
# `swarm-victoriametrics.nix`).
|
||
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
||
|
||
# This host serves the vhost, and the container behind it resolves
|
||
# through the hive's dnsmasq.
|
||
services.hyperhive.gateway.enable = lib.mkDefault true;
|
||
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
||
|
||
# Authenticated front door onto the loopback-only store — see the
|
||
# file-top comment for why this is safe to add without touching the
|
||
# store's own (still unauthenticated, still loopback) listener at all.
|
||
# `forceSSL`: same asymmetry `swarm-ui.nix` documents —
|
||
# authelia answers a plain-http auth subrequest with 400, which
|
||
# `auth_request` cannot read as anything but a broken check, so this
|
||
# vhost needs `forceSSL` rather than the `addSSL` every unauthenticated
|
||
# sibling vhost uses.
|
||
services.hyperhive.gateway.vhosts."${cfg.domain}" = {
|
||
forceSSL = true;
|
||
locations = {
|
||
"/" = {
|
||
proxyPass = "http://127.0.0.1:${toString cfg.port}/";
|
||
extraConfig = swarmAuthRequest;
|
||
};
|
||
|
||
# The swarm collector's ingest route. `=` so it outranks the `/`
|
||
# prefix above — without a more specific location it would ride that
|
||
# catch-all, and that is the whole hazard here.
|
||
#
|
||
# ⚠️ Deliberately NOT `swarmAuthRequest`. That block ends in
|
||
# `error_page 401 =302`, which is right for a browser and wrong for a
|
||
# pusher: handed a redirect it follows the redirect and POSTs its
|
||
# batch at a login page, which answers 200. Ingest then reports
|
||
# healthy while storing nothing. A machine route lets the 401 reach
|
||
# the client unchanged.
|
||
#
|
||
# `auth_request` does not inherit across sibling locations (see the
|
||
# note on `swarmAuthRequest` above), so naming it here is required
|
||
# rather than redundant.
|
||
"= /insert/opentelemetry/v1/logs" = {
|
||
proxyPass = "http://127.0.0.1:${toString cfg.port}/insert/opentelemetry/v1/logs";
|
||
extraConfig = ''
|
||
auth_request /__hive_authelia;
|
||
'';
|
||
};
|
||
|
||
# The read side of that same split: the route a caller holding a
|
||
# bearer token queries, as opposed to the `/` above which is the
|
||
# operator's browser. Bare `auth_request` for the ingest route's
|
||
# exact reason — `error_page 401 =302` hands an unauthenticated
|
||
# machine caller authelia's login page as a **200 with an HTML
|
||
# body**, so a client that reads the status code records a
|
||
# successful query that returned no rows. 401 is the only answer
|
||
# here that a caller cannot mistake for an empty result.
|
||
#
|
||
# `^~` so it outranks the `/` catch-all and stays ahead of any
|
||
# regex location a later change adds. The prefix is the whole
|
||
# LogsQL query API and nothing else — `query`, `tail`, `hits`,
|
||
# `facets`, the `stats_query*` and the `field_*`/`stream_*`
|
||
# routes — while the browser UI sits beside it under
|
||
# `/select/vmui`. Both read off the pinned build, not the docs.
|
||
#
|
||
# The query is forwarded unmodified — no scoping parameter is
|
||
# injected, so any authenticated caller reads the whole swarm's
|
||
# logs. That is the rule in force, not an omission: read
|
||
# permissions are a later thing, and this location is where one
|
||
# attaches when it exists.
|
||
#
|
||
# Three checks: an agent's own `swarm-agent.` bearer goes to
|
||
# `swarm-logs-auth`, Grafana's 64-hex bearer to `swarm-metrics-auth`,
|
||
# anything else to authelia (`appendHttpConfig` above picks).
|
||
"^~ /select/logsql/" = {
|
||
# No URI part, so the request path and its query string reach
|
||
# the store as the caller sent them.
|
||
proxyPass = "http://127.0.0.1:${toString cfg.port}";
|
||
extraConfig = ''
|
||
auth_request /__swarm_logs_auth;
|
||
'';
|
||
};
|
||
|
||
# The store's own health, which it publishes as prometheus metrics
|
||
# on the listener it serves queries on. The swarm collector requests
|
||
# exactly this URL, declared as `publishedScrapeTargets.victorialogs`
|
||
# in ./swarm-otel-service.nix; the two must agree, since that string
|
||
# is also the token's audience.
|
||
#
|
||
# Bare `auth_request` to authelia, as on the ingest route above, and
|
||
# never `/__swarm_logs_auth`, which admits an agent's bearer:
|
||
# authelia's rules for this URL admit the collector's client and
|
||
# deny every other subject (see `metricsRules` in ./swarm-authelia.nix).
|
||
"= /metrics" = {
|
||
proxyPass = "http://127.0.0.1:${toString cfg.port}/metrics";
|
||
extraConfig = ''
|
||
auth_request /__hive_authelia;
|
||
'';
|
||
};
|
||
|
||
# `rewrite … last` re-matches locations, so the subrequest lands in
|
||
# one of the two internal locations named by the map.
|
||
"= /__swarm_logs_auth" = {
|
||
extraConfig = ''
|
||
internal;
|
||
rewrite ^ $swarm_logs_auth_target last;
|
||
'';
|
||
};
|
||
|
||
# The `Authorization` header rides the subrequest unchanged.
|
||
#
|
||
# Any well-formed agent bearer the responder has not recently granted
|
||
# costs a secret store login and read, so the rate per client address
|
||
# is capped here, ahead of the responder. 403 when over: the only
|
||
# statuses `auth_request` passes through are 401 and 403, and
|
||
# `swarm-logs` retries a 401.
|
||
"= /__swarm_logs_agent_auth" = {
|
||
proxyPass = "http://unix:${logsAuthSocket}:/";
|
||
extraConfig = ''
|
||
internal;
|
||
limit_req zone=swarm_logs_agent_auth burst=${toString deployCfg.victorialogs.authBurst} nodelay;
|
||
limit_req_status 403;
|
||
proxy_pass_request_body off;
|
||
proxy_set_header Content-Length "";
|
||
'';
|
||
};
|
||
|
||
# Grafana's bearer, 64 hex digits (`openssl rand -hex 32`), checked by
|
||
# `swarm-metrics-auth`. The responder reads the store at most once per
|
||
# its cache TTL whatever the bearers, so no rate limit sits in front.
|
||
"= /__swarm_logs_grafana_auth" = {
|
||
proxyPass = "http://unix:${grafanaAuthSocket}:/";
|
||
extraConfig = ''
|
||
internal;
|
||
proxy_pass_request_body off;
|
||
proxy_set_header Content-Length "";
|
||
'';
|
||
};
|
||
|
||
# The subrequest itself — same target, same header set, same
|
||
# reasoning as `swarm-ui.nix`'s own copy (measured against the
|
||
# pinned authelia binary, not copied from an example).
|
||
"= /__hive_authelia" = {
|
||
proxyPass = "https://${hyperhiveCfg.swarm.authelia.domain}/api/authz/auth-request";
|
||
# nixpkgs appends its OWN `Host $host` after extraConfig,
|
||
# which would override verifiedProxyTo's — see the comment
|
||
# on verifiedProxyTo in hive-gateway/vhost-lib.nix.
|
||
recommendedProxySettings = false;
|
||
extraConfig = ''
|
||
internal;
|
||
${gatewayCfg.lib.verifiedProxyTo hyperhiveCfg.swarm.authelia.domain}
|
||
proxy_pass_request_body off;
|
||
proxy_set_header Content-Length "";
|
||
proxy_set_header X-Original-Method $request_method;
|
||
proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
|
||
proxy_set_header X-Forwarded-Proto $scheme;
|
||
proxy_set_header X-Forwarded-Host $http_host;
|
||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||
'';
|
||
};
|
||
};
|
||
};
|
||
|
||
# The agents' check on `/select/logsql/`. On the host rather than in the
|
||
# store's container: it needs nothing of the store, only nginx's reach to
|
||
# its socket and its own store identity.
|
||
users.users.${logsAuthUnit} = {
|
||
isSystemUser = true;
|
||
group = logsAuthUnit;
|
||
description = "swarm log store auth_request responder";
|
||
};
|
||
users.groups.${logsAuthUnit} = { };
|
||
|
||
systemd.services.${logsAuthUnit} = {
|
||
description = "swarm log store auth_request responder for agent credentials";
|
||
wantedBy = [ "multi-user.target" ];
|
||
# Where the store's PKI is minted on this host, the leaf is one of its
|
||
# files. Ordering only: elsewhere the unit does not exist.
|
||
after = [
|
||
"network.target"
|
||
"swarm-bao-pki.service"
|
||
];
|
||
serviceConfig = {
|
||
ExecStart = lib.concatStringsSep " " [
|
||
"${deployCfg.victorialogs.authPackage}/bin/swarm-logs-auth"
|
||
"--socket ${logsAuthSocket}"
|
||
# The role ./swarm-bao.nix writes on the store's `cert` mount.
|
||
"--store-cert-role swarm-logs-auth"
|
||
"--verdict-ttl ${toString deployCfg.victorialogs.authVerdictTtl}"
|
||
];
|
||
# Paths only: `argv` is world-readable via /proc/<pid>/cmdline.
|
||
LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") logsAuthStoreFiles;
|
||
# ⚠️ A static user, never `DynamicUser`: with `RuntimeDirectoryPreserve`
|
||
# systemd puts a dynamic user's runtime directory under
|
||
# `/run/private/` (0700 root:root), which nginx cannot traverse.
|
||
User = logsAuthUnit;
|
||
Group = logsAuthUnit;
|
||
RuntimeDirectory = logsAuthUnit;
|
||
# Traverse-only for others, so nginx reaches the socket path.
|
||
RuntimeDirectoryMode = "0751";
|
||
# The path never vanishes from under a running nginx; the responder
|
||
# unlinks a stale socket as it starts.
|
||
RuntimeDirectoryPreserve = "yes";
|
||
Restart = "on-failure";
|
||
RestartSec = "5s";
|
||
SyslogIdentifier = logsAuthUnit;
|
||
ProtectSystem = "strict";
|
||
ProtectHome = true;
|
||
PrivateTmp = "disconnected";
|
||
PrivateDevices = true;
|
||
NoNewPrivileges = true;
|
||
RestrictSUIDSGID = true;
|
||
RemoveIPC = true;
|
||
};
|
||
environment = {
|
||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||
BAO_CLIENT_CERT = logsAuthCredential "bao-client.pem";
|
||
BAO_CLIENT_KEY = logsAuthCredential "bao-client-key.pem";
|
||
}
|
||
// lib.optionalAttrs (logsAuthStoreFiles ? "bao-ca.pem") {
|
||
BAO_CACERT = logsAuthCredential "bao-ca.pem";
|
||
};
|
||
};
|
||
|
||
# Grafana's check on `/select/logsql/`: the metrics store's responder
|
||
# binary, pointed at the bearer Grafana's datasources share.
|
||
users.users.${grafanaAuthUnit} = {
|
||
isSystemUser = true;
|
||
group = grafanaAuthUnit;
|
||
description = "swarm log store auth_request responder for Grafana's bearer";
|
||
};
|
||
users.groups.${grafanaAuthUnit} = { };
|
||
|
||
systemd.services.${grafanaAuthUnit} = {
|
||
description = "swarm log store auth_request responder for Grafana's bearer";
|
||
wantedBy = [ "multi-user.target" ];
|
||
after = [
|
||
"network.target"
|
||
"swarm-bao-pki.service"
|
||
];
|
||
serviceConfig = {
|
||
ExecStart = lib.concatStringsSep " " [
|
||
"${deployCfg.victorialogs.grafanaAuthPackage}/bin/swarm-metrics-auth"
|
||
"--socket ${grafanaAuthSocket}"
|
||
"--store-cert-role swarm-logs-auth"
|
||
"--store-path ${grafanaBearerStorePath}"
|
||
];
|
||
LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") logsAuthStoreFiles;
|
||
# ⚠️ A static user, never `DynamicUser`: see the `swarm-logs-auth` unit above.
|
||
User = grafanaAuthUnit;
|
||
Group = grafanaAuthUnit;
|
||
RuntimeDirectory = grafanaAuthUnit;
|
||
RuntimeDirectoryMode = "0751";
|
||
RuntimeDirectoryPreserve = "yes";
|
||
Restart = "on-failure";
|
||
RestartSec = "5s";
|
||
SyslogIdentifier = grafanaAuthUnit;
|
||
ProtectSystem = "strict";
|
||
ProtectHome = true;
|
||
PrivateTmp = "disconnected";
|
||
PrivateDevices = true;
|
||
NoNewPrivileges = true;
|
||
RestrictSUIDSGID = true;
|
||
RemoveIPC = true;
|
||
};
|
||
environment = {
|
||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||
BAO_CLIENT_CERT = grafanaAuthCredential "bao-client.pem";
|
||
BAO_CLIENT_KEY = grafanaAuthCredential "bao-client-key.pem";
|
||
}
|
||
// lib.optionalAttrs (logsAuthStoreFiles ? "bao-ca.pem") {
|
||
BAO_CACERT = grafanaAuthCredential "bao-ca.pem";
|
||
};
|
||
};
|
||
|
||
containers.${cfg.machine} = {
|
||
autoStart = true;
|
||
ephemeral = false;
|
||
# No `--link-journal=host`: the journal stays in the guest, where the
|
||
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
|
||
extraFlags = lib.mapAttrsToList (
|
||
id: source: "--load-credential=${id}:${source}"
|
||
) forwarderStoreFiles;
|
||
inherit privateNetwork;
|
||
bindMounts = caTrust.bindMount;
|
||
|
||
config =
|
||
{ ... }:
|
||
{
|
||
imports = [
|
||
../container-modules/swarm-container.nix
|
||
../container-modules/swarm-journal-forwarder.nix
|
||
(import ../container-modules/swarm-container-resolver.nix {
|
||
inherit (networkCfg) bridgeIp;
|
||
# The forwarder and its secret fetch dial the swarm collector,
|
||
# authelia and the store by name too.
|
||
dnsConsumers = [
|
||
"victorialogs.service"
|
||
"opentelemetry-collector.service"
|
||
"${forwarderUnit}.service"
|
||
];
|
||
})
|
||
# Go: `SSL_CERT_FILE` replaces its trust store, for the swarm
|
||
# collector's and authelia's names.
|
||
(caTrust.trustBundle {
|
||
inherit pkgs;
|
||
name = cfg.machine;
|
||
consumers = [ "opentelemetry-collector" ];
|
||
})
|
||
];
|
||
|
||
services.hyperhive.swarmContainer = { inherit privateNetwork; };
|
||
|
||
# Authenticated as the forwarder's own OIDC client, with the
|
||
# responder's store identity: one identity for the whole container.
|
||
# It is a system credential, so the fetch unit loads it.
|
||
services.hyperhive.swarmJournalForwarder = {
|
||
enable = true;
|
||
serviceName = "victorialogs";
|
||
endpoint = forwarderEndpoint;
|
||
clientId = forwarderClientId;
|
||
tokenUrl = "${toString hyperhiveCfg.swarm.authelia.url}/api/oidc/token";
|
||
telemetryPort = forwarderTelemetryPort;
|
||
store = {
|
||
package = deployCfg.bao.package;
|
||
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||
secretPath = "secret/swarm/services/${forwarderClientId}/oidc/client";
|
||
clientCertFile = forwarderCredential "bao-client.pem";
|
||
clientKeyFile = forwarderCredential "bao-client-key.pem";
|
||
caFile = if forwarderStoreFiles ? "bao-ca.pem" then forwarderCredential "bao-ca.pem" else null;
|
||
};
|
||
};
|
||
systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames forwarderStoreFiles;
|
||
|
||
services.victorialogs = {
|
||
enable = true;
|
||
package = deployCfg.victorialogs.package;
|
||
|
||
# ⚠️ PINNED TO LOOPBACK: upstream's default listens on every
|
||
# interface, and this endpoint accepts writes as well as reads.
|
||
# Nothing authenticates it either — upstream offers one basic-auth
|
||
# pair (`-httpAuth.username` / `-httpAuth.password`) and this
|
||
# module sets neither. The bind address is the boundary.
|
||
listenAddress = "127.0.0.1:${toString cfg.port}";
|
||
|
||
extraOptions = [ "-retentionPeriod=${deployCfg.victorialogs.retentionPeriod}" ];
|
||
};
|
||
};
|
||
};
|
||
};
|
||
|
||
# 🔑 OTLP ingest is served at `/insert/opentelemetry/v1/logs`, measured
|
||
# against the pinned build rather than read from docs. The path differs
|
||
# from the metrics store's `/opentelemetry/api/v1/push`, so an exporter
|
||
# configured by analogy with the metrics one is silently wrong.
|
||
#
|
||
# ⚠️ And the status code cannot tell you which is which: a wrong path, a
|
||
# right path with a wrong body, and a nonsense path ALL answer 400. Only
|
||
# the server log distinguishes them — the real route complains about the
|
||
# encoding ("json encoding isn't supported ... use protobuf"), while
|
||
# anything else logs "unsupported path requested". Verified with a
|
||
# nonsense-path control, because a probe where every arm returns the same
|
||
# code is not evidence.
|
||
#
|
||
# ⚠️ Like the metrics store's, that endpoint is unauthenticated — which is
|
||
# why `listenAddress` above is loopback and why nothing reaches it except
|
||
# through the gateway, where the ingest route is authenticated. (This module
|
||
# declares a vhost.)
|
||
}
|