ddb583cc22
Ten service modules (forge, matrix chat, authelia, grafana, otel, nats, swarm-ui, victorialogs, victoriametrics, bao UI) wrote services.nginx.virtualHosts themselves, each applying the gateway's listen set, per-name TLS attrs and security headers by hand. They now declare services.hyperhive.gateway.vhosts.<name> with only their own locations and options; ./declared-vhosts.nix renders it into the nginx vhost with listen, TLS and headers added. An eval assertion refuses a declared vhost that sets listen, addSSL, onlySSL, enableACME, useACMEHost, sslCertificate or sslCertificateKey, and one declared while the gateway is off. nats' own leaf is the new `certificate` field. The rendered nginx vhosts are unchanged.
669 lines
30 KiB
Nix
669 lines
30 KiB
Nix
# The swarm's log store: one VictoriaLogs for the whole swarm, in a
|
|
# `swarm-victorialogs` nixos-container beside the metrics store it mirrors.
|
|
#
|
|
# Why a store at all rather than reading journals directly: an agent can
|
|
# verify that a unit was *launched* and never that it is *working*. The
|
|
# container journals are not reachable from an agent, so a diagnosis stops at
|
|
# the first component that is broken — which is precisely the component whose
|
|
# own instrument is least likely to be legible. Collecting logs centrally
|
|
# makes the question answerable without host access.
|
|
#
|
|
# It is the collector that feeds this, not the services directly: one ingest
|
|
# point per swarm, same shape as the metrics path.
|
|
#
|
|
# Gateway vhost is authenticated, unlike the metrics store's: an operator
|
|
# reaches this at `https://${cfg.domain}/` gated by the same `auth_request`
|
|
# check against authelia that swarm-ui's own vhost uses (`swarmAuthRequest`
|
|
# below, same shape as `swarm-ui.nix`'s) — see that file's copy for the full
|
|
# rationale (forceSSL is load-bearing there too: authelia answers a plain-http
|
|
# auth subrequest with 400, which `auth_request` cannot interpret as anything
|
|
# but a broken check). The store's own listener stays loopback-only and
|
|
# unauthenticated, and the vhost is now the ONLY way in from outside: the
|
|
# collector pushes through it too, at its own `= /insert/...` location, because
|
|
# a swarm has one log store and the collector need not share a host with it.
|
|
# ⚠️ That ingest location deliberately does not carry `swarmAuthRequest` — a
|
|
# pusher handed its `error_page 401 =302` follows the redirect and POSTs at a
|
|
# login page, which answers 200. See the location itself.
|
|
#
|
|
# The query API has a machine route of the same shape at `^~ /select/logsql/`,
|
|
# for the same reason, and with no filtering of what an authenticated caller
|
|
# may read — see that location.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.swarm.victorialogs;
|
|
deployCfg = config.services.hyperhive.deploy;
|
|
networkCfg = config.services.hyperhive.network;
|
|
hyperhiveCfg = config.services.hyperhive;
|
|
gatewayCfg = hyperhiveCfg.gateway;
|
|
|
|
# Copy of `swarm-ui.nix`'s own `swarmAuthRequest` — not shared code because
|
|
# this vhost needs exactly the two locations that reference it (`/` and the
|
|
# internal auth-request target below) and pulling in swarm-ui.nix for one
|
|
# string would couple this module to swarm-ui existing at all, which it
|
|
# need not. Same three lines, same reasoning as that file's own comment.
|
|
swarmAuthRequest = ''
|
|
auth_request /__hive_authelia;
|
|
auth_request_set $target_url $scheme://$http_host$request_uri;
|
|
error_page 401 =302 https://${hyperhiveCfg.swarm.authelia.domain}/?rd=$target_url;
|
|
'';
|
|
|
|
# The agents' `auth_request` responder (`swarm-logs-auth`). Its socket's
|
|
# directory holds that socket alone and is traverse-only for others, the
|
|
# shape `swarm-controller.nix` gives its own.
|
|
logsAuthUnit = "swarm-logs-auth";
|
|
logsAuthSocket = "/run/${logsAuthUnit}/auth.sock";
|
|
logsAuthCredential = id: "/run/credentials/${logsAuthUnit}.service/${id}";
|
|
# Credential id → host path of the responder's store identity.
|
|
logsAuthStoreFiles = {
|
|
"bao-client.pem" = deployCfg.victorialogs.authBaoClientCertFile;
|
|
"bao-client-key.pem" = deployCfg.victorialogs.authBaoClientKeyFile;
|
|
}
|
|
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
|
|
"bao-ca.pem" = deployCfg.bao.serverCaFile;
|
|
};
|
|
baoCfg = hyperhiveCfg.swarm.bao;
|
|
|
|
# Grafana's bearer, checked by a second `swarm-metrics-auth` instance. It
|
|
# reads the store with the same leaf as `swarm-logs-auth`, which is granted
|
|
# the one extra path below. The same leaf as `grafanaBearerStorePath` in
|
|
# ./swarm-victoriametrics.nix and `grafanaMetricsBearerLeaf` in ./swarm-bao.nix.
|
|
grafanaAuthUnit = "swarm-logs-grafana-auth";
|
|
grafanaAuthSocket = "/run/${grafanaAuthUnit}/auth.sock";
|
|
grafanaAuthCredential = id: "/run/credentials/${grafanaAuthUnit}.service/${id}";
|
|
grafanaBearerStorePath = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer";
|
|
|
|
# Shared host netns, like every sibling swarm container: Grafana reaches
|
|
# this at 127.0.0.1:<port>.
|
|
privateNetwork = false;
|
|
|
|
# The container's journal forwarder pushes to its own receiver on the swarm
|
|
# collector, the route `serviceProducers.swarm-victorialogs` in
|
|
# ./swarm-otel-service.nix declares. The port suffix is elided on 443, as in
|
|
# ./swarm-bao.nix.
|
|
forwarderClientId = cfg.otel.clientId;
|
|
gatewayPortSuffix = lib.optionalString (
|
|
gatewayCfg.httpsPort != 443
|
|
) ":${toString gatewayCfg.httpsPort}";
|
|
forwarderEndpoint = "https://${hyperhiveCfg.swarm.otel.domain}${gatewayPortSuffix}/${cfg.machine}";
|
|
forwarderUnit = "swarm-journal-forwarder-oidc";
|
|
# In the host netns: distinct from every other collector's there, which the
|
|
# assertion below checks.
|
|
forwarderTelemetryPort = 8897;
|
|
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
|
|
# The responder's store identity, handed into the container as system
|
|
# credentials for the forwarder's fetch unit. The host units keep reading
|
|
# the same files.
|
|
forwarderStoreFiles = lib.filterAttrs (_: source: source != null) logsAuthStoreFiles;
|
|
# nixos-container expands `extraFlags` unquoted, so a source path must be
|
|
# one word.
|
|
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
|
|
|
|
# The forwarder reaches the swarm collector and authelia by name over TLS; in
|
|
# self-signed mode their leaves chain to the hive CA.
|
|
caTrust = import ./lib/hive-ca-trust.nix {
|
|
inherit lib gatewayCfg;
|
|
tlsCfg = deployCfg.hive-controller.tls;
|
|
};
|
|
in
|
|
{
|
|
# What the store IS from any hive's point of view, the name it answers on and
|
|
# the port, is `swarm.victorialogs` in ./swarm-victorialogs-service.nix.
|
|
# `enable`, `package` and `retentionPeriod` are decisions of the host that
|
|
# runs it and live under `deploy.*`.
|
|
|
|
# Retention is a property of the store this host runs, not something the
|
|
# swarm has to agree on: it is read only where the container is defined,
|
|
# and a hive that is a *client* of the log store never consults it. That
|
|
# makes it a `deploy.*` value by the same rule as the seal on the secret
|
|
# store — options on the auto-deployed service itself.
|
|
options.services.hyperhive.deploy.victorialogs.package = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = pkgs.victorialogs;
|
|
defaultText = lib.literalExpression "pkgs.victorialogs";
|
|
description = "VictoriaLogs package to run.";
|
|
};
|
|
|
|
options.services.hyperhive.deploy.victorialogs.retentionPeriod = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "30d";
|
|
example = "90d";
|
|
description = ''
|
|
How long log data is kept.
|
|
|
|
Deliberately far shorter than the metrics store's retention: logs
|
|
are orders of magnitude larger per unit of time, and their value
|
|
decays much faster. A log line answers "what happened during that
|
|
incident"; a metric answers "is this worse than last quarter".
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.deploy.victorialogs.authPackage = lib.mkOption {
|
|
type = lib.types.package;
|
|
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-logs-auth";
|
|
description = ''
|
|
The `swarm-logs-auth` package: the gateway's `auth_request` responder
|
|
that checks an agent's `swarm-agent.<agent>.<secret>` bearer on
|
|
`/select/logsql/` against the secret store.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.deploy.victorialogs.authBaoClientCertFile = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
example = "/var/lib/swarm-bao-pki/logs-auth.pem";
|
|
description = ''
|
|
Client certificate `swarm-logs-auth` presents to the swarm's secret store
|
|
to read agents' queue secrets. Its subject must be
|
|
{option}`services.hyperhive.deploy.bao.logsAuthCommonName`.
|
|
|
|
Required when `deploy.victorialogs.enable` is set; unset fails at eval.
|
|
./glue-logs-auth-bao-identity.nix points it at the leaf
|
|
./glue-bao-tls.nix mints on a host that runs the store; on any other
|
|
log-store host it is a file the operator places.
|
|
|
|
A path, never a value.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.deploy.victorialogs.authBaoClientKeyFile = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
example = "/var/lib/swarm-bao-pki/logs-auth-key.pem";
|
|
description = ''
|
|
Private key for {option}`services.hyperhive.deploy.victorialogs.authBaoClientCertFile`.
|
|
A path, never a value.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.deploy.victorialogs.authVerdictTtl = lib.mkOption {
|
|
type = lib.types.ints.unsigned;
|
|
default = 60;
|
|
description = ''
|
|
Seconds `swarm-logs-auth` grants an agent bearer it has verified again
|
|
without reading the secret store. A denial is never remembered.
|
|
|
|
This is also how long a re-minted or deleted queue secret keeps working
|
|
at the log store's query route after its last successful check. `0`
|
|
checks the store on every query.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.deploy.victorialogs.authRate = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "50r/s";
|
|
example = "30r/m";
|
|
description = ''
|
|
nginx `limit_req` rate, per client address, of `/select/logsql/`
|
|
requests carrying a `swarm-agent.` bearer, which nginx checks with
|
|
`swarm-logs-auth`. Each such request that the responder has not granted
|
|
within {option}`services.hyperhive.deploy.victorialogs.authVerdictTtl`
|
|
costs a secret store login and read, whether the bearer is right or not.
|
|
|
|
Counts every agent query, granted or not, and is keyed by client address,
|
|
so every agent on one host shares one budget. The default covers about a
|
|
hundred agents on a host cold-starting at once, each costing one store
|
|
login and read for its first query. Requests over the rate and its burst
|
|
are answered 403.
|
|
'';
|
|
};
|
|
|
|
options.services.hyperhive.deploy.victorialogs.authBurst = lib.mkOption {
|
|
type = lib.types.ints.positive;
|
|
default = 500;
|
|
description = ''
|
|
Requests a client address may make above
|
|
{option}`services.hyperhive.deploy.victorialogs.authRate` before nginx
|
|
answers 403, served without delay.
|
|
'';
|
|
};
|
|
|
|
config = lib.mkIf deployCfg.victorialogs.enable {
|
|
assertions = [
|
|
{
|
|
assertion =
|
|
deployCfg.victorialogs.authBaoClientCertFile != null
|
|
&& deployCfg.victorialogs.authBaoClientKeyFile != null;
|
|
message = ''
|
|
services.hyperhive.deploy.victorialogs.authBaoClientCertFile and
|
|
authBaoClientKeyFile must both be set.
|
|
|
|
swarm-logs-auth reads agents' queue secrets from the swarm's secret
|
|
store with this certificate. They default to the leaf the store
|
|
mints where services.hyperhive.deploy.bao.enable is set on this
|
|
host; elsewhere, issue a leaf whose CN is
|
|
services.hyperhive.deploy.bao.logsAuthCommonName and name it in
|
|
both options.
|
|
'';
|
|
}
|
|
{
|
|
assertion = lib.all oneWord (lib.attrValues forwarderStoreFiles);
|
|
message = ''
|
|
The log store responder's store identity paths must not contain
|
|
whitespace: nspawn's --load-credential receives them through
|
|
nixos-container's unquoted extra flags.
|
|
'';
|
|
}
|
|
{
|
|
assertion =
|
|
(hyperhiveCfg.swarm.otel.serviceProducers.${cfg.machine}.clientId or null) == forwarderClientId;
|
|
message = ''
|
|
services.hyperhive.swarm.otel.serviceProducers has no entry
|
|
"${cfg.machine}" with clientId "${forwarderClientId}", so the swarm
|
|
collector has no receiver for the log store's journal forwarder.
|
|
Declare it in swarm-otel-service.nix.
|
|
'';
|
|
}
|
|
{
|
|
assertion =
|
|
!(lib.elem forwarderTelemetryPort [
|
|
hyperhiveCfg.otel.telemetryPort
|
|
hyperhiveCfg.swarm.otel.telemetryPort
|
|
hyperhiveCfg.swarm.bao.otel.telemetryPort
|
|
hyperhiveCfg.swarm.nats.otel.telemetryPort
|
|
]);
|
|
message = ''
|
|
The log store's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
|
|
collides with another collector's in the host netns: the hive
|
|
collector's, the swarm collector's, the store forwarder's or the
|
|
queue forwarder's.
|
|
'';
|
|
}
|
|
];
|
|
|
|
# nspawn fails to start the container when a credential source is
|
|
# missing, so where this host mints the leaf the container pulls the
|
|
# minting unit in and waits for it, as ./swarm-builder.nix's does. The CA
|
|
# ordering keeps the trust bundle's bind source there first.
|
|
systemd.services."container@${cfg.machine}" = lib.mkMerge [
|
|
caTrust.containerOrdering
|
|
{
|
|
after = [ "swarm-bao-pki.service" ];
|
|
wants = [ "swarm-bao-pki.service" ];
|
|
}
|
|
];
|
|
|
|
# Which check a `/select/logsql/` request gets. `auth_request` takes a
|
|
# fixed URI, so it names `/__swarm_logs_auth`, which rewrites to the
|
|
# target this map picks from the caller's `Authorization`.
|
|
services.nginx.appendHttpConfig = ''
|
|
map $http_authorization $swarm_logs_auth_target {
|
|
"~^Bearer swarm-agent\." /__swarm_logs_agent_auth;
|
|
"~^Bearer [0-9a-f]{64}$" /__swarm_logs_grafana_auth;
|
|
default /__hive_authelia;
|
|
}
|
|
limit_req_zone $binary_remote_addr zone=swarm_logs_agent_auth:1m rate=${deployCfg.victorialogs.authRate};
|
|
'';
|
|
|
|
# The gateway name, inside `deployCfg.victorialogs.enable` — same
|
|
# "only the host that runs the service may claim the name" guard every
|
|
# sibling swarm-service module uses (`swarm-grafana.nix`,
|
|
# `swarm-victoriametrics.nix`).
|
|
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
|
|
|
# This host serves the vhost, and the container behind it resolves
|
|
# through the hive's dnsmasq.
|
|
services.hyperhive.gateway.enable = lib.mkDefault true;
|
|
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
|
|
|
# Authenticated front door onto the loopback-only store — see the
|
|
# file-top comment for why this is safe to add without touching the
|
|
# store's own (still unauthenticated, still loopback) listener at all.
|
|
# `forceSSL`: same asymmetry `swarm-ui.nix` documents —
|
|
# authelia answers a plain-http auth subrequest with 400, which
|
|
# `auth_request` cannot read as anything but a broken check, so this
|
|
# vhost needs `forceSSL` rather than the `addSSL` every unauthenticated
|
|
# sibling vhost uses.
|
|
services.hyperhive.gateway.vhosts."${cfg.domain}" = {
|
|
forceSSL = true;
|
|
locations = {
|
|
"/" = {
|
|
proxyPass = "http://127.0.0.1:${toString cfg.port}/";
|
|
extraConfig = swarmAuthRequest;
|
|
};
|
|
|
|
# The swarm collector's ingest route. `=` so it outranks the `/`
|
|
# prefix above — without a more specific location it would ride that
|
|
# catch-all, and that is the whole hazard here.
|
|
#
|
|
# ⚠️ Deliberately NOT `swarmAuthRequest`. That block ends in
|
|
# `error_page 401 =302`, which is right for a browser and wrong for a
|
|
# pusher: handed a redirect it follows the redirect and POSTs its
|
|
# batch at a login page, which answers 200. Ingest then reports
|
|
# healthy while storing nothing. A machine route lets the 401 reach
|
|
# the client unchanged.
|
|
#
|
|
# `auth_request` does not inherit across sibling locations (see the
|
|
# note on `swarmAuthRequest` above), so naming it here is required
|
|
# rather than redundant.
|
|
"= /insert/opentelemetry/v1/logs" = {
|
|
proxyPass = "http://127.0.0.1:${toString cfg.port}/insert/opentelemetry/v1/logs";
|
|
extraConfig = ''
|
|
auth_request /__hive_authelia;
|
|
'';
|
|
};
|
|
|
|
# The read side of that same split: the route a caller holding a
|
|
# bearer token queries, as opposed to the `/` above which is the
|
|
# operator's browser. Bare `auth_request` for the ingest route's
|
|
# exact reason — `error_page 401 =302` hands an unauthenticated
|
|
# machine caller authelia's login page as a **200 with an HTML
|
|
# body**, so a client that reads the status code records a
|
|
# successful query that returned no rows. 401 is the only answer
|
|
# here that a caller cannot mistake for an empty result.
|
|
#
|
|
# `^~` so it outranks the `/` catch-all and stays ahead of any
|
|
# regex location a later change adds. The prefix is the whole
|
|
# LogsQL query API and nothing else — `query`, `tail`, `hits`,
|
|
# `facets`, the `stats_query*` and the `field_*`/`stream_*`
|
|
# routes — while the browser UI sits beside it under
|
|
# `/select/vmui`. Both read off the pinned build, not the docs.
|
|
#
|
|
# The query is forwarded unmodified — no scoping parameter is
|
|
# injected, so any authenticated caller reads the whole swarm's
|
|
# logs. That is the rule in force, not an omission: read
|
|
# permissions are a later thing, and this location is where one
|
|
# attaches when it exists.
|
|
#
|
|
# Three checks: an agent's own `swarm-agent.` bearer goes to
|
|
# `swarm-logs-auth`, Grafana's 64-hex bearer to `swarm-metrics-auth`,
|
|
# anything else to authelia (`appendHttpConfig` above picks).
|
|
"^~ /select/logsql/" = {
|
|
# No URI part, so the request path and its query string reach
|
|
# the store as the caller sent them.
|
|
proxyPass = "http://127.0.0.1:${toString cfg.port}";
|
|
extraConfig = ''
|
|
auth_request /__swarm_logs_auth;
|
|
'';
|
|
};
|
|
|
|
# The store's own health, which it publishes as prometheus metrics
|
|
# on the listener it serves queries on. The swarm collector requests
|
|
# exactly this URL, declared as `publishedScrapeTargets.victorialogs`
|
|
# in ./swarm-otel-service.nix; the two must agree, since that string
|
|
# is also the token's audience.
|
|
#
|
|
# Bare `auth_request` to authelia, as on the ingest route above, and
|
|
# never `/__swarm_logs_auth`, which admits an agent's bearer:
|
|
# authelia's rules for this URL admit the collector's client and
|
|
# deny every other subject (see `metricsRules` in ./swarm-authelia.nix).
|
|
"= /metrics" = {
|
|
proxyPass = "http://127.0.0.1:${toString cfg.port}/metrics";
|
|
extraConfig = ''
|
|
auth_request /__hive_authelia;
|
|
'';
|
|
};
|
|
|
|
# `rewrite … last` re-matches locations, so the subrequest lands in
|
|
# one of the two internal locations named by the map.
|
|
"= /__swarm_logs_auth" = {
|
|
extraConfig = ''
|
|
internal;
|
|
rewrite ^ $swarm_logs_auth_target last;
|
|
'';
|
|
};
|
|
|
|
# The `Authorization` header rides the subrequest unchanged.
|
|
#
|
|
# Any well-formed agent bearer the responder has not recently granted
|
|
# costs a secret store login and read, so the rate per client address
|
|
# is capped here, ahead of the responder. 403 when over: the only
|
|
# statuses `auth_request` passes through are 401 and 403, and
|
|
# `swarm-logs` retries a 401.
|
|
"= /__swarm_logs_agent_auth" = {
|
|
proxyPass = "http://unix:${logsAuthSocket}:/";
|
|
extraConfig = ''
|
|
internal;
|
|
limit_req zone=swarm_logs_agent_auth burst=${toString deployCfg.victorialogs.authBurst} nodelay;
|
|
limit_req_status 403;
|
|
proxy_pass_request_body off;
|
|
proxy_set_header Content-Length "";
|
|
'';
|
|
};
|
|
|
|
# Grafana's bearer, 64 hex digits (`openssl rand -hex 32`), checked by
|
|
# `swarm-metrics-auth`. The responder reads the store at most once per
|
|
# its cache TTL whatever the bearers, so no rate limit sits in front.
|
|
"= /__swarm_logs_grafana_auth" = {
|
|
proxyPass = "http://unix:${grafanaAuthSocket}:/";
|
|
extraConfig = ''
|
|
internal;
|
|
proxy_pass_request_body off;
|
|
proxy_set_header Content-Length "";
|
|
'';
|
|
};
|
|
|
|
# The subrequest itself — same target, same header set, same
|
|
# reasoning as `swarm-ui.nix`'s own copy (measured against the
|
|
# pinned authelia binary, not copied from an example).
|
|
"= /__hive_authelia" = {
|
|
proxyPass = "https://${hyperhiveCfg.swarm.authelia.domain}/api/authz/auth-request";
|
|
# nixpkgs appends its OWN `Host $host` after extraConfig,
|
|
# which would override verifiedProxyTo's — see the comment
|
|
# on verifiedProxyTo in hive-gateway/vhost-lib.nix.
|
|
recommendedProxySettings = false;
|
|
extraConfig = ''
|
|
internal;
|
|
${gatewayCfg.lib.verifiedProxyTo hyperhiveCfg.swarm.authelia.domain}
|
|
proxy_pass_request_body off;
|
|
proxy_set_header Content-Length "";
|
|
proxy_set_header X-Original-Method $request_method;
|
|
proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
|
|
proxy_set_header X-Forwarded-Proto $scheme;
|
|
proxy_set_header X-Forwarded-Host $http_host;
|
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
|
'';
|
|
};
|
|
};
|
|
};
|
|
|
|
# The agents' check on `/select/logsql/`. On the host rather than in the
|
|
# store's container: it needs nothing of the store, only nginx's reach to
|
|
# its socket and its own store identity.
|
|
users.users.${logsAuthUnit} = {
|
|
isSystemUser = true;
|
|
group = logsAuthUnit;
|
|
description = "swarm log store auth_request responder";
|
|
};
|
|
users.groups.${logsAuthUnit} = { };
|
|
|
|
systemd.services.${logsAuthUnit} = {
|
|
description = "swarm log store auth_request responder for agent credentials";
|
|
wantedBy = [ "multi-user.target" ];
|
|
# Where the store's PKI is minted on this host, the leaf is one of its
|
|
# files. Ordering only: elsewhere the unit does not exist.
|
|
after = [
|
|
"network.target"
|
|
"swarm-bao-pki.service"
|
|
];
|
|
serviceConfig = {
|
|
ExecStart = lib.concatStringsSep " " [
|
|
"${deployCfg.victorialogs.authPackage}/bin/swarm-logs-auth"
|
|
"--socket ${logsAuthSocket}"
|
|
# The role ./swarm-bao.nix writes on the store's `cert` mount.
|
|
"--store-cert-role swarm-logs-auth"
|
|
"--verdict-ttl ${toString deployCfg.victorialogs.authVerdictTtl}"
|
|
];
|
|
# Paths only: `argv` is world-readable via /proc/<pid>/cmdline.
|
|
LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") logsAuthStoreFiles;
|
|
# ⚠️ A static user, never `DynamicUser`: with `RuntimeDirectoryPreserve`
|
|
# systemd puts a dynamic user's runtime directory under
|
|
# `/run/private/` (0700 root:root), which nginx cannot traverse.
|
|
User = logsAuthUnit;
|
|
Group = logsAuthUnit;
|
|
RuntimeDirectory = logsAuthUnit;
|
|
# Traverse-only for others, so nginx reaches the socket path.
|
|
RuntimeDirectoryMode = "0751";
|
|
# The path never vanishes from under a running nginx; the responder
|
|
# unlinks a stale socket as it starts.
|
|
RuntimeDirectoryPreserve = "yes";
|
|
Restart = "on-failure";
|
|
RestartSec = "5s";
|
|
SyslogIdentifier = logsAuthUnit;
|
|
ProtectSystem = "strict";
|
|
ProtectHome = true;
|
|
PrivateTmp = "disconnected";
|
|
PrivateDevices = true;
|
|
NoNewPrivileges = true;
|
|
RestrictSUIDSGID = true;
|
|
RemoveIPC = true;
|
|
};
|
|
environment = {
|
|
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
|
BAO_CLIENT_CERT = logsAuthCredential "bao-client.pem";
|
|
BAO_CLIENT_KEY = logsAuthCredential "bao-client-key.pem";
|
|
}
|
|
// lib.optionalAttrs (logsAuthStoreFiles ? "bao-ca.pem") {
|
|
BAO_CACERT = logsAuthCredential "bao-ca.pem";
|
|
};
|
|
};
|
|
|
|
# Grafana's check on `/select/logsql/`: the metrics store's responder
|
|
# binary, pointed at the bearer Grafana's datasources share.
|
|
users.users.${grafanaAuthUnit} = {
|
|
isSystemUser = true;
|
|
group = grafanaAuthUnit;
|
|
description = "swarm log store auth_request responder for Grafana's bearer";
|
|
};
|
|
users.groups.${grafanaAuthUnit} = { };
|
|
|
|
systemd.services.${grafanaAuthUnit} = {
|
|
description = "swarm log store auth_request responder for Grafana's bearer";
|
|
wantedBy = [ "multi-user.target" ];
|
|
after = [
|
|
"network.target"
|
|
"swarm-bao-pki.service"
|
|
];
|
|
serviceConfig = {
|
|
ExecStart = lib.concatStringsSep " " [
|
|
"${deployCfg.victoriametrics.authPackage}/bin/swarm-metrics-auth"
|
|
"--socket ${grafanaAuthSocket}"
|
|
"--store-cert-role swarm-logs-auth"
|
|
"--store-path ${grafanaBearerStorePath}"
|
|
];
|
|
LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") logsAuthStoreFiles;
|
|
# ⚠️ A static user, never `DynamicUser`: see the `swarm-logs-auth` unit above.
|
|
User = grafanaAuthUnit;
|
|
Group = grafanaAuthUnit;
|
|
RuntimeDirectory = grafanaAuthUnit;
|
|
RuntimeDirectoryMode = "0751";
|
|
RuntimeDirectoryPreserve = "yes";
|
|
Restart = "on-failure";
|
|
RestartSec = "5s";
|
|
SyslogIdentifier = grafanaAuthUnit;
|
|
ProtectSystem = "strict";
|
|
ProtectHome = true;
|
|
PrivateTmp = "disconnected";
|
|
PrivateDevices = true;
|
|
NoNewPrivileges = true;
|
|
RestrictSUIDSGID = true;
|
|
RemoveIPC = true;
|
|
};
|
|
environment = {
|
|
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
|
BAO_CLIENT_CERT = grafanaAuthCredential "bao-client.pem";
|
|
BAO_CLIENT_KEY = grafanaAuthCredential "bao-client-key.pem";
|
|
}
|
|
// lib.optionalAttrs (logsAuthStoreFiles ? "bao-ca.pem") {
|
|
BAO_CACERT = grafanaAuthCredential "bao-ca.pem";
|
|
};
|
|
};
|
|
|
|
containers.${cfg.machine} = {
|
|
autoStart = true;
|
|
ephemeral = false;
|
|
# No `--link-journal=host`: the journal stays in the guest, where the
|
|
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
|
|
extraFlags = lib.mapAttrsToList (
|
|
id: source: "--load-credential=${id}:${source}"
|
|
) forwarderStoreFiles;
|
|
inherit privateNetwork;
|
|
bindMounts = caTrust.bindMount;
|
|
|
|
config =
|
|
{ ... }:
|
|
{
|
|
imports = [
|
|
../container-modules/swarm-container.nix
|
|
../container-modules/swarm-journal-forwarder.nix
|
|
(import ../container-modules/swarm-container-resolver.nix {
|
|
inherit (networkCfg) bridgeIp;
|
|
# The forwarder and its secret fetch dial the swarm collector,
|
|
# authelia and the store by name too.
|
|
dnsConsumers = [
|
|
"victorialogs.service"
|
|
"opentelemetry-collector.service"
|
|
"${forwarderUnit}.service"
|
|
];
|
|
})
|
|
# Go: `SSL_CERT_FILE` replaces its trust store, for the swarm
|
|
# collector's and authelia's names.
|
|
(caTrust.trustBundle {
|
|
inherit pkgs;
|
|
name = cfg.machine;
|
|
consumers = [ "opentelemetry-collector" ];
|
|
})
|
|
];
|
|
|
|
services.hyperhive.swarmContainer = { inherit privateNetwork; };
|
|
|
|
# Authenticated as the forwarder's own OIDC client, with the
|
|
# responder's store identity: one identity for the whole container.
|
|
# It is a system credential, so the fetch unit loads it.
|
|
services.hyperhive.swarmJournalForwarder = {
|
|
enable = true;
|
|
serviceName = "victorialogs";
|
|
endpoint = forwarderEndpoint;
|
|
clientId = forwarderClientId;
|
|
tokenUrl = "${toString hyperhiveCfg.swarm.authelia.url}/api/oidc/token";
|
|
telemetryPort = forwarderTelemetryPort;
|
|
store = {
|
|
package = deployCfg.bao.package;
|
|
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
|
secretPath = "secret/swarm/services/${forwarderClientId}/oidc/client";
|
|
clientCertFile = forwarderCredential "bao-client.pem";
|
|
clientKeyFile = forwarderCredential "bao-client-key.pem";
|
|
caFile = if forwarderStoreFiles ? "bao-ca.pem" then forwarderCredential "bao-ca.pem" else null;
|
|
};
|
|
};
|
|
systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames forwarderStoreFiles;
|
|
|
|
services.victorialogs = {
|
|
enable = true;
|
|
package = deployCfg.victorialogs.package;
|
|
|
|
# ⚠️ PINNED TO LOOPBACK: upstream's default listens on every
|
|
# interface, and this endpoint accepts writes as well as reads.
|
|
# Nothing authenticates it either — upstream offers one basic-auth
|
|
# pair (`-httpAuth.username` / `-httpAuth.password`) and this
|
|
# module sets neither. The bind address is the boundary.
|
|
listenAddress = "127.0.0.1:${toString cfg.port}";
|
|
|
|
extraOptions = [ "-retentionPeriod=${deployCfg.victorialogs.retentionPeriod}" ];
|
|
};
|
|
};
|
|
};
|
|
};
|
|
|
|
# 🔑 OTLP ingest is served at `/insert/opentelemetry/v1/logs`, measured
|
|
# against the pinned build rather than read from docs. The path differs
|
|
# from the metrics store's `/opentelemetry/api/v1/push`, so an exporter
|
|
# configured by analogy with the metrics one is silently wrong.
|
|
#
|
|
# ⚠️ And the status code cannot tell you which is which: a wrong path, a
|
|
# right path with a wrong body, and a nonsense path ALL answer 400. Only
|
|
# the server log distinguishes them — the real route complains about the
|
|
# encoding ("json encoding isn't supported ... use protobuf"), while
|
|
# anything else logs "unsupported path requested". Verified with a
|
|
# nonsense-path control, because a probe where every arm returns the same
|
|
# code is not evidence.
|
|
#
|
|
# ⚠️ Like the metrics store's, that endpoint is unauthenticated — which is
|
|
# why `listenAddress` above is loopback and why nothing reaches it except
|
|
# through the gateway, where the ingest route is authenticated. (This module
|
|
# declares a vhost.)
|
|
}
|