vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-otel: scrape nats through its published name

Browse source 
The queue's metrics were scraped over loopback from the exporter, declared
only where the queue and the collector share a host, so a collector on any
other host had no job for it. Its target is now declared on every host as
`https://<swarm.nats.domain>/metrics`, and the queue's host gets a gateway
vhost on that name serving `= /metrics` from the exporter behind the
collector's auth subrequest (`/__nats_metrics_authz`, the forge's shape).
Every other path there is 403; the monitoring endpoint is never proxied.
The rules derived from `publishedScrapeTargets` admit the collector's
client and deny every other subject. The loopback entry, and with it
`glue-otel-scrape-targets.nix`, is gone.

In self-signed mode the vhost serves the queue's own `swarm-nats` leaf
(issued for exactly that name, `server_flag` set), copied 0640
root:nginx by `swarm-nats-gateway-cert`; until the leaf exists that unit
writes a throwaway pair, so nginx never starts on a missing file. The
name stays out of `swarm.serviceDomains`, so a host whose only swarm
service is the queue fronts no services name and its gateway neither
waits for nor needs the swarm-services leaf: no services-issuer pair has
to be placed on it. With ACME or an operator cert the vhost uses the
gateway's own TLS, as every other name does.

The queue's host now enables the gateway (`mkDefault`), as every other
swarm service's host does.

Fixes #5205
This commit is contained in:
atlas 2026-10-14 20:21:31 +02:00
commit 217b3f1cea
9 changed files with 276 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

134
nix/host-modules/swarm-nats.nix
View file

@ -286,6 +286,14 @@ let
tlsKeyCredential = "tls-key";
tlsKeyCredentialPath = "/run/credentials/nats.service/${tlsKeyCredential}";
# The gateway's copy of that leaf, for the `/metrics` vhost below: the key
# here is 0640 root:nginx, so nginx reads it while the queue's own stays
# 0600 root. Self-signed mode only, the mode `gateway.lib.tlsFor` hands the
# swarm-services leaf to; with ACME or an operator cert that issuer covers
# this name as it covers every other.
gatewayServesQueueLeaf = gatewayCfg.enable && gatewayCfg.useSelfSigned;
gatewayTlsDir = "/var/lib/swarm-nats-gateway-tls";
# The responder's own store identity, for the callout xkey, agent queue
# credentials and its OIDC client secret. Required: an assertion below refuses
# a queue without it.
@ -329,12 +337,12 @@ in
# The swarm's message queue: one NATS server, reached by every hive at
# `tls://<swarm.nats.domain>:<port>`.
#
# ⚠️ There is deliberately NO gateway vhost here. NATS speaks its own TCP
# protocol rather than HTTP, so nginx cannot front it the way it fronts the
# forge, matrix and authelia, and the server terminates TLS itself. The
# ⚠️ The gateway does NOT front the queue. NATS speaks its own TCP protocol
# rather than HTTP, so the server terminates TLS itself on `port`. The
# name is the half of the sibling pattern that applies, the way it is for
# bao: `gateway.localNames` on this host, the operator's DNS everywhere
# else.
# else. The one gateway vhost on that name serves the exporter's
# `/metrics` to the swarm collector and nothing else.
# What the queue IS to every hive, the ports it answers on and the client id
# it is registered under, is `swarm.nats` in ./swarm-nats-service.nix. What
@ -429,14 +437,124 @@ in
};
config = lib.mkIf deployCfg.nats.enable {
# Resolver only: NATS speaks its own protocol, so nginx fronts
# nothing here — but the auth responder introspects authelia by name.
# The auth responder introspects authelia by name, and the gateway serves
# the exporter's `/metrics` below.
services.hyperhive.gateway.enable = lib.mkDefault true;
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
# The name every client dials, answered with the bridge IP on this host.
# Other hives resolve it through the operator's DNS, as they do bao's.
services.hyperhive.gateway.localNames = [ cfg.domain ];
# The exporter's metrics, for the swarm collector wherever it runs, and
# nothing else on this name: the monitoring endpoint (`/varz`, `/connz`)
# is never proxied, and every other path is 403.
#
# ⚠️ Not `gateway.lib.tlsFor`, and the name is not in
# `swarm.serviceDomains`. In self-signed mode this vhost serves the
# queue's own leaf (issued for exactly this name, `server_flag` set), so
# a host whose only swarm service is the queue never fronts a services
# name and its gateway does not wait for the swarm-services leaf.
#
# `forceSSL` for the reason ./swarm-victorialogs.nix gives: authelia
# answers a plain-http auth subrequest with 400.
services.nginx.virtualHosts."${cfg.domain}" =
(
if gatewayServesQueueLeaf then
{
sslCertificate = "${gatewayTlsDir}/cert.pem";
sslCertificateKey = "${gatewayTlsDir}/key.pem";
}
else
builtins.removeAttrs (gatewayCfg.lib.tlsFor cfg.domain) [ "addSSL" ]
)
// {
forceSSL = true;
listen = gatewayCfg.lib.listen;
extraConfig = gatewayCfg.lib.securityHeaders;
locations = {
"/".return = "403";
# The collector requests exactly this URL, declared as
# `publishedScrapeTargets.nats` in ./swarm-otel-service.nix; the
# two must agree, since that string is also the token's audience.
# Authelia's rules for it admit the collector's client and deny
# every other subject (`metricsRules` in ./swarm-authelia.nix).
"= /metrics" = {
proxyPass = "http://127.0.0.1:${toString cfg.metricsPort}/metrics";
extraConfig = ''
auth_request /__nats_metrics_authz;
'';
};
# The subrequest, same shape as the forge's `= /__forge_metrics_authz`
# (./hive-forge/default.nix): no `error_page 401 =302`, so a scraper
# gets the 401 rather than a login page parsed as metrics.
"= /__nats_metrics_authz" = {
proxyPass = "https://${autheliaCfg.domain}/api/authz/auth-request";
# nixpkgs appends its OWN `Host $host` after extraConfig, which
# would override verifiedProxyTo's — see hive-gateway/vhost-lib.nix.
recommendedProxySettings = false;
extraConfig = ''
internal;
${gatewayCfg.lib.verifiedProxyTo autheliaCfg.domain}
proxy_pass_request_body off;
proxy_set_header Content-Length "";
proxy_set_header X-Original-Method $request_method;
proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
'';
};
};
};
# Puts a pair at the paths the vhost above names, before nginx starts:
# nginx refuses a config naming a missing cert file, and that takes the
# whole gateway down rather than this one name. The queue's leaf when it
# exists, else a throwaway self-signed pair until `swarm-bao-nats-tls`
# issues the leaf and starts this again. Never fails on a missing leaf.
systemd.services.swarm-nats-gateway-cert = lib.mkIf gatewayServesQueueLeaf {
description = "Copy the swarm queue's TLS leaf for the gateway's /metrics vhost";
wantedBy = [ "nginx.service" ];
before = [ "nginx.service" ];
path = [
pkgs.coreutils
pkgs.openssl
pkgs.systemd
];
serviceConfig = {
Type = "oneshot";
UMask = "0077";
SyslogIdentifier = "swarm-nats-gateway-cert";
};
script = ''
set -eu
src=${lib.escapeShellArg tlsDir}
d=${lib.escapeShellArg gatewayTlsDir}
install -d -m 0755 "$d"
if [ -s "$src/cert.pem" ] && [ -s "$src/key.pem" ]; then
cp "$src/cert.pem" "$d/cert.pem.new"
cp "$src/key.pem" "$d/key.pem.new"
elif [ -s "$d/cert.pem" ] && [ -s "$d/key.pem" ]; then
exit 0
else
echo "no queue leaf yet — serving a throwaway self-signed certificate on ${cfg.domain} until swarm-bao-nats-tls issues one" >&2
openssl req -x509 -newkey rsa:2048 -nodes -sha256 -days 30 \
-subj "/CN=${cfg.domain}" -keyout "$d/key.pem.new" -out "$d/cert.pem.new"
fi
chmod 0644 "$d/cert.pem.new"
chgrp nginx "$d/key.pem.new"
chmod 0640 "$d/key.pem.new"
mv -f "$d/key.pem.new" "$d/key.pem"
mv -f "$d/cert.pem.new" "$d/cert.pem"
if systemctl is-active --quiet nginx.service; then
systemctl reload nginx.service
fi
'';
};
assertions = [
{
# The callout xkey and the responder's OIDC client secret are read
@ -999,6 +1117,10 @@ in
chmod 0644 "$d/cert.pem.new"
mv -f "$d/key.pem.new" "$d/key.pem"
mv -f "$d/cert.pem.new" "$d/cert.pem"
${lib.optionalString gatewayServesQueueLeaf ''
# The gateway's `/metrics` vhost serves a copy of this leaf.
systemctl start swarm-nats-gateway-cert.service''}
# Renewal, or the late-store retry. On a normal boot the container is
# not up yet and starts the server with this leaf itself. A restart,