From 217b3f1cea1e0202490a796f899a934bd3ff26f4 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 14 Oct 2026 20:21:31 +0200 Subject: [PATCH] swarm-otel: scrape nats through its published name The queue's metrics were scraped over loopback from the exporter, declared only where the queue and the collector share a host, so a collector on any other host had no job for it. Its target is now declared on every host as `https:///metrics`, and the queue's host gets a gateway vhost on that name serving `= /metrics` from the exporter behind the collector's auth subrequest (`/__nats_metrics_authz`, the forge's shape). Every other path there is 403; the monitoring endpoint is never proxied. The rules derived from `publishedScrapeTargets` admit the collector's client and deny every other subject. The loopback entry, and with it `glue-otel-scrape-targets.nix`, is gone. In self-signed mode the vhost serves the queue's own `swarm-nats` leaf (issued for exactly that name, `server_flag` set), copied 0640 root:nginx by `swarm-nats-gateway-cert`; until the leaf exists that unit writes a throwaway pair, so nginx never starts on a missing file. The name stays out of `swarm.serviceDomains`, so a host whose only swarm service is the queue fronts no services name and its gateway neither waits for nor needs the swarm-services leaf: no services-issuer pair has to be placed on it. With ACME or an operator cert the vhost uses the gateway's own TLS, as every other name does. The queue's host now enables the gateway (`mkDefault`), as every other swarm service's host does. Fixes #5205 --- nix/host-modules/default.nix | 1 - nix/host-modules/glue-otel-scrape-targets.nix | 37 ----- nix/host-modules/swarm-nats-service.nix | 11 +- nix/host-modules/swarm-nats.nix | 134 +++++++++++++++++- nix/host-modules/swarm-otel-service.nix | 6 +- nix/host-modules/swarm.nix | 4 + nix/module-eval/forge-placement.nix | 11 +- nix/module-eval/nats-tls.nix | 68 +++++++++ nix/module-eval/swarm-otel-identity.nix | 60 +++++++- 9 files changed, 276 insertions(+), 56 deletions(-) delete mode 100644 nix/host-modules/glue-otel-scrape-targets.nix diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index cf88a4f5..ba169276 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -44,7 +44,6 @@ ./glue-nats-bao-identity.nix ./glue-nats-oidc-client.nix ./glue-nats-wireguard.nix - ./glue-otel-scrape-targets.nix ./glue-queue-agent-credential.nix ./glue-builder-oidc-client.nix ./glue-review-classifier-oidc-client.nix diff --git a/nix/host-modules/glue-otel-scrape-targets.nix b/nix/host-modules/glue-otel-scrape-targets.nix deleted file mode 100644 index e01e8204..00000000 --- a/nix/host-modules/glue-otel-scrape-targets.nix +++ /dev/null @@ -1,37 +0,0 @@ -# Glue: declare the loopback scrape target of nats on the swarm's OTEL -# collector, wherever it shares a host with it. Authelia and grafana are -# `publishedScrapeTargets` jobs (./swarm-otel-service.nix); a loopback job of -# the same name fails ./swarm-otel.nix's job-name assertion. -# -# ONE PAIRING — nats ← the collector. Deleting this leaves the collector not -# scraping nats: no error, no metrics. -# -# The target is gated on the service AND the collector being enabled here. The -# loopback address is only honest when both run on this host; a target nobody -# can read asserts a collection that is not happening. Both default from -# `deploy.allSwarmServices`, but `mkDefault` is an invitation to override, so -# co-location is a property of the auto-deployed topology, not the modules. -# -# ⚠️ Silent when split: separate hosts are separate evaluations with no shared -# context, so a host running nats and no collector declares nothing and -# the collector's host never learns of it. No error, no warning — a -# healthy exporter and an empty dashboard. Not guardable from here. -# -# ⚠️ `swarm.otel`, not `hyperhive.otel` — two collectors one word apart, and only -# this one reads `scrapeTargets`. -{ - lib, - config, - ... -}: -let - hyperhiveCfg = config.services.hyperhive; - deployCfg = hyperhiveCfg.deploy; - swarmCfg = hyperhiveCfg.swarm; - withCollector = service: lib.mkIf (deployCfg.${service}.enable && deployCfg.swarm-otel.enable); -in -{ - config.services.hyperhive.swarm.otel.scrapeTargets = { - nats = withCollector "nats" "127.0.0.1:${toString swarmCfg.nats.metricsPort}"; - }; -} diff --git a/nix/host-modules/swarm-nats-service.nix b/nix/host-modules/swarm-nats-service.nix index 56ec72cf..aeeea857 100644 --- a/nix/host-modules/swarm-nats-service.nix +++ b/nix/host-modules/swarm-nats-service.nix @@ -40,8 +40,11 @@ in Every other hive resolves it through the operator's upstream DNS, which is where a multi-host swarm needs a record for it. - Not in {option}`services.hyperhive.swarm.serviceDomains`: that list - is the names a gateway vhost fronts, and nginx fronts nothing here. + Not in {option}`services.hyperhive.swarm.serviceDomains`. The gateway + on the queue's host serves `https:///metrics` to the swarm + collector, and in self-signed mode it serves the queue's own leaf + there rather than the swarm-services one, so that host never waits + for a services leaf (./swarm-nats.nix). ''; }; @@ -89,7 +92,9 @@ in default = 7777; description = '' Port the Prometheus exporter serves NATS's metrics on, bound to - loopback for the swarm collector to scrape. + loopback. The gateway's `= /metrics` location on + {option}`services.hyperhive.swarm.nats.domain` proxies it, behind + the swarm collector's authelia check. ⚠️ This and {option}`monitorPort` are two more claims on a port space every swarm container shares — they run in this container but diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index 5bc0d64a..2f285e52 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -286,6 +286,14 @@ let tlsKeyCredential = "tls-key"; tlsKeyCredentialPath = "/run/credentials/nats.service/${tlsKeyCredential}"; + # The gateway's copy of that leaf, for the `/metrics` vhost below: the key + # here is 0640 root:nginx, so nginx reads it while the queue's own stays + # 0600 root. Self-signed mode only, the mode `gateway.lib.tlsFor` hands the + # swarm-services leaf to; with ACME or an operator cert that issuer covers + # this name as it covers every other. + gatewayServesQueueLeaf = gatewayCfg.enable && gatewayCfg.useSelfSigned; + gatewayTlsDir = "/var/lib/swarm-nats-gateway-tls"; + # The responder's own store identity, for the callout xkey, agent queue # credentials and its OIDC client secret. Required: an assertion below refuses # a queue without it. @@ -329,12 +337,12 @@ in # The swarm's message queue: one NATS server, reached by every hive at # `tls://:`. # - # ⚠️ There is deliberately NO gateway vhost here. NATS speaks its own TCP - # protocol rather than HTTP, so nginx cannot front it the way it fronts the - # forge, matrix and authelia, and the server terminates TLS itself. The + # ⚠️ The gateway does NOT front the queue. NATS speaks its own TCP protocol + # rather than HTTP, so the server terminates TLS itself on `port`. The # name is the half of the sibling pattern that applies, the way it is for # bao: `gateway.localNames` on this host, the operator's DNS everywhere - # else. + # else. The one gateway vhost on that name serves the exporter's + # `/metrics` to the swarm collector and nothing else. # What the queue IS to every hive, the ports it answers on and the client id # it is registered under, is `swarm.nats` in ./swarm-nats-service.nix. What @@ -429,14 +437,124 @@ in }; config = lib.mkIf deployCfg.nats.enable { - # Resolver only: NATS speaks its own protocol, so nginx fronts - # nothing here — but the auth responder introspects authelia by name. + # The auth responder introspects authelia by name, and the gateway serves + # the exporter's `/metrics` below. + services.hyperhive.gateway.enable = lib.mkDefault true; services.hyperhive.gateway.dns.enable = lib.mkDefault true; # The name every client dials, answered with the bridge IP on this host. # Other hives resolve it through the operator's DNS, as they do bao's. services.hyperhive.gateway.localNames = [ cfg.domain ]; + # The exporter's metrics, for the swarm collector wherever it runs, and + # nothing else on this name: the monitoring endpoint (`/varz`, `/connz`) + # is never proxied, and every other path is 403. + # + # ⚠️ Not `gateway.lib.tlsFor`, and the name is not in + # `swarm.serviceDomains`. In self-signed mode this vhost serves the + # queue's own leaf (issued for exactly this name, `server_flag` set), so + # a host whose only swarm service is the queue never fronts a services + # name and its gateway does not wait for the swarm-services leaf. + # + # `forceSSL` for the reason ./swarm-victorialogs.nix gives: authelia + # answers a plain-http auth subrequest with 400. + services.nginx.virtualHosts."${cfg.domain}" = + ( + if gatewayServesQueueLeaf then + { + sslCertificate = "${gatewayTlsDir}/cert.pem"; + sslCertificateKey = "${gatewayTlsDir}/key.pem"; + } + else + builtins.removeAttrs (gatewayCfg.lib.tlsFor cfg.domain) [ "addSSL" ] + ) + // { + forceSSL = true; + listen = gatewayCfg.lib.listen; + extraConfig = gatewayCfg.lib.securityHeaders; + locations = { + "/".return = "403"; + + # The collector requests exactly this URL, declared as + # `publishedScrapeTargets.nats` in ./swarm-otel-service.nix; the + # two must agree, since that string is also the token's audience. + # Authelia's rules for it admit the collector's client and deny + # every other subject (`metricsRules` in ./swarm-authelia.nix). + "= /metrics" = { + proxyPass = "http://127.0.0.1:${toString cfg.metricsPort}/metrics"; + extraConfig = '' + auth_request /__nats_metrics_authz; + ''; + }; + + # The subrequest, same shape as the forge's `= /__forge_metrics_authz` + # (./hive-forge/default.nix): no `error_page 401 =302`, so a scraper + # gets the 401 rather than a login page parsed as metrics. + "= /__nats_metrics_authz" = { + proxyPass = "https://${autheliaCfg.domain}/api/authz/auth-request"; + # nixpkgs appends its OWN `Host $host` after extraConfig, which + # would override verifiedProxyTo's — see hive-gateway/vhost-lib.nix. + recommendedProxySettings = false; + extraConfig = '' + internal; + ${gatewayCfg.lib.verifiedProxyTo autheliaCfg.domain} + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-Method $request_method; + proxy_set_header X-Original-URL $scheme://$http_host$request_uri; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + ''; + }; + }; + }; + + # Puts a pair at the paths the vhost above names, before nginx starts: + # nginx refuses a config naming a missing cert file, and that takes the + # whole gateway down rather than this one name. The queue's leaf when it + # exists, else a throwaway self-signed pair until `swarm-bao-nats-tls` + # issues the leaf and starts this again. Never fails on a missing leaf. + systemd.services.swarm-nats-gateway-cert = lib.mkIf gatewayServesQueueLeaf { + description = "Copy the swarm queue's TLS leaf for the gateway's /metrics vhost"; + wantedBy = [ "nginx.service" ]; + before = [ "nginx.service" ]; + path = [ + pkgs.coreutils + pkgs.openssl + pkgs.systemd + ]; + serviceConfig = { + Type = "oneshot"; + UMask = "0077"; + SyslogIdentifier = "swarm-nats-gateway-cert"; + }; + script = '' + set -eu + src=${lib.escapeShellArg tlsDir} + d=${lib.escapeShellArg gatewayTlsDir} + install -d -m 0755 "$d" + if [ -s "$src/cert.pem" ] && [ -s "$src/key.pem" ]; then + cp "$src/cert.pem" "$d/cert.pem.new" + cp "$src/key.pem" "$d/key.pem.new" + elif [ -s "$d/cert.pem" ] && [ -s "$d/key.pem" ]; then + exit 0 + else + echo "no queue leaf yet — serving a throwaway self-signed certificate on ${cfg.domain} until swarm-bao-nats-tls issues one" >&2 + openssl req -x509 -newkey rsa:2048 -nodes -sha256 -days 30 \ + -subj "/CN=${cfg.domain}" -keyout "$d/key.pem.new" -out "$d/cert.pem.new" + fi + chmod 0644 "$d/cert.pem.new" + chgrp nginx "$d/key.pem.new" + chmod 0640 "$d/key.pem.new" + mv -f "$d/key.pem.new" "$d/key.pem" + mv -f "$d/cert.pem.new" "$d/cert.pem" + if systemctl is-active --quiet nginx.service; then + systemctl reload nginx.service + fi + ''; + }; + assertions = [ { # The callout xkey and the responder's OIDC client secret are read @@ -999,6 +1117,10 @@ in chmod 0644 "$d/cert.pem.new" mv -f "$d/key.pem.new" "$d/key.pem" mv -f "$d/cert.pem.new" "$d/cert.pem" + ${lib.optionalString gatewayServesQueueLeaf '' + + # The gateway's `/metrics` vhost serves a copy of this leaf. + systemctl start swarm-nats-gateway-cert.service''} # Renewal, or the late-store retry. On a normal boot the container is # not up yet and starts the server with this leaf itself. A restart, diff --git a/nix/host-modules/swarm-otel-service.nix b/nix/host-modules/swarm-otel-service.nix index 63b992fb..9b7edcf0 100644 --- a/nix/host-modules/swarm-otel-service.nix +++ b/nix/host-modules/swarm-otel-service.nix @@ -240,8 +240,8 @@ in declared on a host that does not serve it. An entry that also needs the collector on the same host (the loopback address is only honest then) is declared from a glue module instead, which reads both - switches: grafana and nats are declared by - `glue-otel-scrape-targets.nix`. + switches. Every swarm service is a `publishedScrapeTargets` entry + instead, which a collector on any host scrapes. ⚠️ **That constrains the TARGET, not the SCRAPER, and the difference is a silent gap.** Nothing here puts the collector on the same host. @@ -364,5 +364,7 @@ in victorialogs = "https://${hyperhiveCfg.swarm.victorialogs.domain}/metrics"; # The vhost location is in ./swarm-grafana.nix. grafana = "https://${hyperhiveCfg.swarm.grafana.domain}/metrics"; + # The vhost location is in ./swarm-nats.nix. + nats = "https://${hyperhiveCfg.swarm.nats.domain}/metrics"; }; } diff --git a/nix/host-modules/swarm.nix b/nix/host-modules/swarm.nix index e91b780c..ee393eba 100644 --- a/nix/host-modules/swarm.nix +++ b/nix/host-modules/swarm.nix @@ -93,6 +93,10 @@ let # services leaf over the hive one, so a gateway-published swarm name # missing here is served the hive certificate, which cannot cover a name # under a different apex. + # + # `swarm.nats.domain` is deliberately absent: its `/metrics` vhost names + # the queue's own leaf instead (./swarm-nats.nix), so a host running only + # the queue fronts no services name and never waits for that leaf. serviceDomains' = [ swarmCfg.forge.domain swarmCfg.matrix.gatewayHost diff --git a/nix/module-eval/forge-placement.nix b/nix/module-eval/forge-placement.nix index f04b5dcd..49579f65 100644 --- a/nix/module-eval/forge-placement.nix +++ b/nix/module-eval/forge-placement.nix @@ -138,15 +138,16 @@ let && !(refusedOver bare "deploy.forgejo.sso.clientSecretFile"); } { - # Authelia and grafana are published targets; ./swarm-otel-identity.nix - # covers them. - name = "a host running the collector, authelia, grafana and nats scrapes nats on loopback, authelia and grafana not"; + # All three are published targets; ./swarm-otel-identity.nix covers them. + name = "a host running the collector, authelia, grafana and nats scrapes none of them on loopback"; ok = let t = servicesHere.services.hyperhive.swarm.otel.scrapeTargets; - port = n: toString servicesHere.services.hyperhive.swarm.${n}.metricsPort; in - t.nats == "127.0.0.1:${port "nats"}" && !(t ? authelia) && !(t ? grafana); + servicesHere.services.hyperhive.deploy.nats.enable + && !(t ? authelia) + && !(t ? grafana) + && !(t ? nats); } { name = "hosting the swarm's shared services runs the forge"; diff --git a/nix/module-eval/nats-tls.nix b/nix/module-eval/nats-tls.nix index c0301447..b64d3695 100644 --- a/nix/module-eval/nats-tls.nix +++ b/nix/module-eval/nats-tls.nix @@ -362,6 +362,74 @@ let failedAssertions remoteLeafNoUrl == [ ] && !(remoteLeafNoUrl.systemd.services.hive-c0re.environment ? HIVE_C0RE_NATS_URL); } + { + # Only `/metrics`, from the exporter, behind the collector's subrequest: + # no `error_page` (a scraper would parse a login page as metrics), the + # monitoring endpoint never proxied, every other path refused. In + # self-signed mode the vhost names the queue's own leaf, as copied for + # nginx, not the swarm-services one. + name = "the queue's name serves only /metrics, behind the collector's auth subrequest, on the queue's own leaf"; + ok = + let + h = allLocal.services.hyperhive; + v = allLocal.services.nginx.virtualHosts.${natsName}; + locs = v.locations; + checksOf = l: lib.filter (lib.hasPrefix "auth_request") (lib.splitString "\n" l.extraConfig); + authz = locs."= /__nats_metrics_authz"; + in + lib.attrNames locs == [ + "/" + "= /__nats_metrics_authz" + "= /metrics" + ] + && locs."/".return == "403" + && locs."/".proxyPass == null + && locs."= /metrics".proxyPass == "http://127.0.0.1:${toString h.swarm.nats.metricsPort}/metrics" + && checksOf locs."= /metrics" == [ "auth_request /__nats_metrics_authz;" ] + && !(lib.hasInfix "error_page" locs."= /metrics".extraConfig) + && authz.proxyPass == "https://auth.t.local/api/authz/auth-request" + && lib.hasInfix "internal;" authz.extraConfig + && !(lib.any ( + l: + lib.hasInfix ":${toString h.swarm.nats.monitorPort}" ( + if l.proxyPass == null then "" else l.proxyPass + ) + ) (lib.attrValues locs)) + && v.forceSSL + && v.sslCertificate == "/var/lib/swarm-nats-gateway-tls/cert.pem" + && v.sslCertificateKey == "/var/lib/swarm-nats-gateway-tls/key.pem"; + } + { + # A host whose only swarm service is the queue: its gateway serves the + # queue's name with the queue's leaf, fronts no swarm-services name, and + # so neither waits for nor requires the swarm-services leaf. The copy + # unit is only wanted by nginx, never required, and a rotated queue leaf + # re-runs it. + name = "a host running only the queue renders its gateway without requiring the swarm-services leaf"; + ok = + let + h = queueNoStore.services.hyperhive; + s = queueNoStore.systemd.services; + copy = s.swarm-nats-gateway-cert; + in + h.deploy.nats.enable + && !h.deploy.hive-controller.enable + && h.gateway.enable + && h.gateway.useSelfSigned + && + queueNoStore.services.nginx.virtualHosts.${natsName}.sslCertificate + == "/var/lib/swarm-nats-gateway-tls/cert.pem" + && h.swarm.localServiceDomains == [ ] + && !(lib.elem natsName h.swarm.serviceDomains) + && !(lib.elem "hive-gateway-self-signed-cert.service" (s.swarm-services-cert.requiredBy or [ ])) + && !(lib.hasInfix "serves swarm names only with it" s.hive-gateway-self-signed-cert.script) + && lib.elem "nginx.service" copy.wantedBy + && lib.elem "nginx.service" copy.before + && !(lib.elem "nginx.service" (copy.requiredBy or [ ])) + && lib.hasInfix "throwaway self-signed" copy.script + && lib.hasInfix "chgrp nginx" copy.script + && lib.hasInfix "systemctl start swarm-nats-gateway-cert.service" s.swarm-bao-nats-tls.script; + } ]; in runGroup "nats-tls" cases diff --git a/nix/module-eval/swarm-otel-identity.nix b/nix/module-eval/swarm-otel-identity.nix index 04c0d047..2d27d63c 100644 --- a/nix/module-eval/swarm-otel-identity.nix +++ b/nix/module-eval/swarm-otel-identity.nix @@ -111,6 +111,7 @@ let grafanaMetricsUrl = "https://grafana.t.local/metrics"; logsMetricsUrl = "https://logs.t.local/metrics"; vmMetricsUrl = "https://metrics.t.local/metrics"; + natsMetricsUrl = "https://nats.t.local/metrics"; # The policy authelia applies to `subject` requesting `https://`, # where `path` is what authelia matches `resources` against: the cleaned path @@ -378,8 +379,9 @@ let grafanaMetricsUrl logsMetricsUrl vmMetricsUrl + natsMetricsUrl ] - && lib.length targets == 10 + && lib.length targets == 12 && lib.all (t: policy collector t == "one_factor") targets && lib.all (t: policy "user:someone" t == "deny") targets && lib.all (t: policy "group:admins" t == "deny") targets; @@ -427,7 +429,7 @@ let ]; matched = row: builtins.match re row.path != null; in - lib.length metricsRules == 8 + lib.length metricsRules == 10 && lib.all (r: r.resources == [ re ]) metricsRules && re == "^/metrics([?].*)?$" && lib.all (row: matched row == (row.nginx == "= /metrics")) table; @@ -621,6 +623,60 @@ let && lib.length jobs == 1 && (lib.head jobs).static_configs == [ { targets = [ "grafana.t.local" ]; } ]; } + { + name = "a collector on a host without the queue scrapes it by name"; + ok = + let + s = otelSettings otelRemoteAuthelia; + job = lib.findFirst (j: j.job_name == "nats") null s.receivers.prometheus.config.scrape_configs; + in + !otelRemoteAuthelia.services.hyperhive.deploy.nats.enable + && job != null + && job.static_configs == [ { targets = [ "nats.t.local" ]; } ] + && job.metrics_path == "/metrics" + && job.oauth2.endpoint_params.audience == natsMetricsUrl; + } + { + # The `deny` that confines the queue's `/metrics` is scoped to that path, + # and comes after the collector's rule. + name = "authelia confines the queue's /metrics to the collector"; + ok = + let + h = autheliaOnly.services.hyperhive; + collector = "oauth2:client:${h.swarm.otel.clientId}"; + rules = + autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules; + natsRules = lib.filter (r: r.domain == "nats.t.local") rules; + policy = subject: autheliaPolicy autheliaOnly subject "nats.t.local"; + in + map (r: r.policy) natsRules == [ + "one_factor" + "deny" + ] + && lib.all (r: r.resources == [ "^/metrics([?].*)?$" ]) natsRules + && (lib.head natsRules).subject == [ collector ] + && policy collector "/metrics" == "one_factor" + && policy "user:someone" "/metrics" == "deny" + && policy "user:someone" "/metrics?x=1" == "deny" + && policy "group:admins" "/metrics" == "deny"; + } + { + # Single host: the queue is scraped once, by its published name. + name = "a host running the queue and the collector scrapes it once, by name"; + ok = + let + o = allLocal.services.hyperhive.swarm.otel; + jobs = + lib.filter (j: j.job_name == "nats") + (otelSettings allLocal).receivers.prometheus.config.scrape_configs; + in + allLocal.services.hyperhive.deploy.nats.enable + && allLocal.services.hyperhive.deploy.swarm-otel.enable + && !(o.scrapeTargets ? nats) + && (o.publishedScrapeTargets.nats or null) == natsMetricsUrl + && lib.length jobs == 1 + && (lib.head jobs).static_configs == [ { targets = [ "nats.t.local" ]; } ]; + } { # Read against the roster the fixture declares rather than against # names spelled here: an arm naming `h1` passes on a single-hive