diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index cf88a4f5..ba169276 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -44,7 +44,6 @@ ./glue-nats-bao-identity.nix ./glue-nats-oidc-client.nix ./glue-nats-wireguard.nix - ./glue-otel-scrape-targets.nix ./glue-queue-agent-credential.nix ./glue-builder-oidc-client.nix ./glue-review-classifier-oidc-client.nix diff --git a/nix/host-modules/glue-otel-scrape-targets.nix b/nix/host-modules/glue-otel-scrape-targets.nix deleted file mode 100644 index e01e8204..00000000 --- a/nix/host-modules/glue-otel-scrape-targets.nix +++ /dev/null @@ -1,37 +0,0 @@ -# Glue: declare the loopback scrape target of nats on the swarm's OTEL -# collector, wherever it shares a host with it. Authelia and grafana are -# `publishedScrapeTargets` jobs (./swarm-otel-service.nix); a loopback job of -# the same name fails ./swarm-otel.nix's job-name assertion. -# -# ONE PAIRING — nats ← the collector. Deleting this leaves the collector not -# scraping nats: no error, no metrics. -# -# The target is gated on the service AND the collector being enabled here. The -# loopback address is only honest when both run on this host; a target nobody -# can read asserts a collection that is not happening. Both default from -# `deploy.allSwarmServices`, but `mkDefault` is an invitation to override, so -# co-location is a property of the auto-deployed topology, not the modules. -# -# ⚠️ Silent when split: separate hosts are separate evaluations with no shared -# context, so a host running nats and no collector declares nothing and -# the collector's host never learns of it. No error, no warning — a -# healthy exporter and an empty dashboard. Not guardable from here. -# -# ⚠️ `swarm.otel`, not `hyperhive.otel` — two collectors one word apart, and only -# this one reads `scrapeTargets`. -{ - lib, - config, - ... -}: -let - hyperhiveCfg = config.services.hyperhive; - deployCfg = hyperhiveCfg.deploy; - swarmCfg = hyperhiveCfg.swarm; - withCollector = service: lib.mkIf (deployCfg.${service}.enable && deployCfg.swarm-otel.enable); -in -{ - config.services.hyperhive.swarm.otel.scrapeTargets = { - nats = withCollector "nats" "127.0.0.1:${toString swarmCfg.nats.metricsPort}"; - }; -} diff --git a/nix/host-modules/swarm-nats-service.nix b/nix/host-modules/swarm-nats-service.nix index 56ec72cf..aeeea857 100644 --- a/nix/host-modules/swarm-nats-service.nix +++ b/nix/host-modules/swarm-nats-service.nix @@ -40,8 +40,11 @@ in Every other hive resolves it through the operator's upstream DNS, which is where a multi-host swarm needs a record for it. - Not in {option}`services.hyperhive.swarm.serviceDomains`: that list - is the names a gateway vhost fronts, and nginx fronts nothing here. + Not in {option}`services.hyperhive.swarm.serviceDomains`. The gateway + on the queue's host serves `https:///metrics` to the swarm + collector, and in self-signed mode it serves the queue's own leaf + there rather than the swarm-services one, so that host never waits + for a services leaf (./swarm-nats.nix). ''; }; @@ -89,7 +92,9 @@ in default = 7777; description = '' Port the Prometheus exporter serves NATS's metrics on, bound to - loopback for the swarm collector to scrape. + loopback. The gateway's `= /metrics` location on + {option}`services.hyperhive.swarm.nats.domain` proxies it, behind + the swarm collector's authelia check. ⚠️ This and {option}`monitorPort` are two more claims on a port space every swarm container shares — they run in this container but diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index 5bc0d64a..2f285e52 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -286,6 +286,14 @@ let tlsKeyCredential = "tls-key"; tlsKeyCredentialPath = "/run/credentials/nats.service/${tlsKeyCredential}"; + # The gateway's copy of that leaf, for the `/metrics` vhost below: the key + # here is 0640 root:nginx, so nginx reads it while the queue's own stays + # 0600 root. Self-signed mode only, the mode `gateway.lib.tlsFor` hands the + # swarm-services leaf to; with ACME or an operator cert that issuer covers + # this name as it covers every other. + gatewayServesQueueLeaf = gatewayCfg.enable && gatewayCfg.useSelfSigned; + gatewayTlsDir = "/var/lib/swarm-nats-gateway-tls"; + # The responder's own store identity, for the callout xkey, agent queue # credentials and its OIDC client secret. Required: an assertion below refuses # a queue without it. @@ -329,12 +337,12 @@ in # The swarm's message queue: one NATS server, reached by every hive at # `tls://:`. # - # ⚠️ There is deliberately NO gateway vhost here. NATS speaks its own TCP - # protocol rather than HTTP, so nginx cannot front it the way it fronts the - # forge, matrix and authelia, and the server terminates TLS itself. The + # ⚠️ The gateway does NOT front the queue. NATS speaks its own TCP protocol + # rather than HTTP, so the server terminates TLS itself on `port`. The # name is the half of the sibling pattern that applies, the way it is for # bao: `gateway.localNames` on this host, the operator's DNS everywhere - # else. + # else. The one gateway vhost on that name serves the exporter's + # `/metrics` to the swarm collector and nothing else. # What the queue IS to every hive, the ports it answers on and the client id # it is registered under, is `swarm.nats` in ./swarm-nats-service.nix. What @@ -429,14 +437,124 @@ in }; config = lib.mkIf deployCfg.nats.enable { - # Resolver only: NATS speaks its own protocol, so nginx fronts - # nothing here — but the auth responder introspects authelia by name. + # The auth responder introspects authelia by name, and the gateway serves + # the exporter's `/metrics` below. + services.hyperhive.gateway.enable = lib.mkDefault true; services.hyperhive.gateway.dns.enable = lib.mkDefault true; # The name every client dials, answered with the bridge IP on this host. # Other hives resolve it through the operator's DNS, as they do bao's. services.hyperhive.gateway.localNames = [ cfg.domain ]; + # The exporter's metrics, for the swarm collector wherever it runs, and + # nothing else on this name: the monitoring endpoint (`/varz`, `/connz`) + # is never proxied, and every other path is 403. + # + # ⚠️ Not `gateway.lib.tlsFor`, and the name is not in + # `swarm.serviceDomains`. In self-signed mode this vhost serves the + # queue's own leaf (issued for exactly this name, `server_flag` set), so + # a host whose only swarm service is the queue never fronts a services + # name and its gateway does not wait for the swarm-services leaf. + # + # `forceSSL` for the reason ./swarm-victorialogs.nix gives: authelia + # answers a plain-http auth subrequest with 400. + services.nginx.virtualHosts."${cfg.domain}" = + ( + if gatewayServesQueueLeaf then + { + sslCertificate = "${gatewayTlsDir}/cert.pem"; + sslCertificateKey = "${gatewayTlsDir}/key.pem"; + } + else + builtins.removeAttrs (gatewayCfg.lib.tlsFor cfg.domain) [ "addSSL" ] + ) + // { + forceSSL = true; + listen = gatewayCfg.lib.listen; + extraConfig = gatewayCfg.lib.securityHeaders; + locations = { + "/".return = "403"; + + # The collector requests exactly this URL, declared as + # `publishedScrapeTargets.nats` in ./swarm-otel-service.nix; the + # two must agree, since that string is also the token's audience. + # Authelia's rules for it admit the collector's client and deny + # every other subject (`metricsRules` in ./swarm-authelia.nix). + "= /metrics" = { + proxyPass = "http://127.0.0.1:${toString cfg.metricsPort}/metrics"; + extraConfig = '' + auth_request /__nats_metrics_authz; + ''; + }; + + # The subrequest, same shape as the forge's `= /__forge_metrics_authz` + # (./hive-forge/default.nix): no `error_page 401 =302`, so a scraper + # gets the 401 rather than a login page parsed as metrics. + "= /__nats_metrics_authz" = { + proxyPass = "https://${autheliaCfg.domain}/api/authz/auth-request"; + # nixpkgs appends its OWN `Host $host` after extraConfig, which + # would override verifiedProxyTo's — see hive-gateway/vhost-lib.nix. + recommendedProxySettings = false; + extraConfig = '' + internal; + ${gatewayCfg.lib.verifiedProxyTo autheliaCfg.domain} + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-Method $request_method; + proxy_set_header X-Original-URL $scheme://$http_host$request_uri; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + ''; + }; + }; + }; + + # Puts a pair at the paths the vhost above names, before nginx starts: + # nginx refuses a config naming a missing cert file, and that takes the + # whole gateway down rather than this one name. The queue's leaf when it + # exists, else a throwaway self-signed pair until `swarm-bao-nats-tls` + # issues the leaf and starts this again. Never fails on a missing leaf. + systemd.services.swarm-nats-gateway-cert = lib.mkIf gatewayServesQueueLeaf { + description = "Copy the swarm queue's TLS leaf for the gateway's /metrics vhost"; + wantedBy = [ "nginx.service" ]; + before = [ "nginx.service" ]; + path = [ + pkgs.coreutils + pkgs.openssl + pkgs.systemd + ]; + serviceConfig = { + Type = "oneshot"; + UMask = "0077"; + SyslogIdentifier = "swarm-nats-gateway-cert"; + }; + script = '' + set -eu + src=${lib.escapeShellArg tlsDir} + d=${lib.escapeShellArg gatewayTlsDir} + install -d -m 0755 "$d" + if [ -s "$src/cert.pem" ] && [ -s "$src/key.pem" ]; then + cp "$src/cert.pem" "$d/cert.pem.new" + cp "$src/key.pem" "$d/key.pem.new" + elif [ -s "$d/cert.pem" ] && [ -s "$d/key.pem" ]; then + exit 0 + else + echo "no queue leaf yet — serving a throwaway self-signed certificate on ${cfg.domain} until swarm-bao-nats-tls issues one" >&2 + openssl req -x509 -newkey rsa:2048 -nodes -sha256 -days 30 \ + -subj "/CN=${cfg.domain}" -keyout "$d/key.pem.new" -out "$d/cert.pem.new" + fi + chmod 0644 "$d/cert.pem.new" + chgrp nginx "$d/key.pem.new" + chmod 0640 "$d/key.pem.new" + mv -f "$d/key.pem.new" "$d/key.pem" + mv -f "$d/cert.pem.new" "$d/cert.pem" + if systemctl is-active --quiet nginx.service; then + systemctl reload nginx.service + fi + ''; + }; + assertions = [ { # The callout xkey and the responder's OIDC client secret are read @@ -999,6 +1117,10 @@ in chmod 0644 "$d/cert.pem.new" mv -f "$d/key.pem.new" "$d/key.pem" mv -f "$d/cert.pem.new" "$d/cert.pem" + ${lib.optionalString gatewayServesQueueLeaf '' + + # The gateway's `/metrics` vhost serves a copy of this leaf. + systemctl start swarm-nats-gateway-cert.service''} # Renewal, or the late-store retry. On a normal boot the container is # not up yet and starts the server with this leaf itself. A restart, diff --git a/nix/host-modules/swarm-otel-service.nix b/nix/host-modules/swarm-otel-service.nix index 63b992fb..9b7edcf0 100644 --- a/nix/host-modules/swarm-otel-service.nix +++ b/nix/host-modules/swarm-otel-service.nix @@ -240,8 +240,8 @@ in declared on a host that does not serve it. An entry that also needs the collector on the same host (the loopback address is only honest then) is declared from a glue module instead, which reads both - switches: grafana and nats are declared by - `glue-otel-scrape-targets.nix`. + switches. Every swarm service is a `publishedScrapeTargets` entry + instead, which a collector on any host scrapes. ⚠️ **That constrains the TARGET, not the SCRAPER, and the difference is a silent gap.** Nothing here puts the collector on the same host. @@ -364,5 +364,7 @@ in victorialogs = "https://${hyperhiveCfg.swarm.victorialogs.domain}/metrics"; # The vhost location is in ./swarm-grafana.nix. grafana = "https://${hyperhiveCfg.swarm.grafana.domain}/metrics"; + # The vhost location is in ./swarm-nats.nix. + nats = "https://${hyperhiveCfg.swarm.nats.domain}/metrics"; }; } diff --git a/nix/host-modules/swarm.nix b/nix/host-modules/swarm.nix index e91b780c..ee393eba 100644 --- a/nix/host-modules/swarm.nix +++ b/nix/host-modules/swarm.nix @@ -93,6 +93,10 @@ let # services leaf over the hive one, so a gateway-published swarm name # missing here is served the hive certificate, which cannot cover a name # under a different apex. + # + # `swarm.nats.domain` is deliberately absent: its `/metrics` vhost names + # the queue's own leaf instead (./swarm-nats.nix), so a host running only + # the queue fronts no services name and never waits for that leaf. serviceDomains' = [ swarmCfg.forge.domain swarmCfg.matrix.gatewayHost diff --git a/nix/module-eval/forge-placement.nix b/nix/module-eval/forge-placement.nix index f04b5dcd..49579f65 100644 --- a/nix/module-eval/forge-placement.nix +++ b/nix/module-eval/forge-placement.nix @@ -138,15 +138,16 @@ let && !(refusedOver bare "deploy.forgejo.sso.clientSecretFile"); } { - # Authelia and grafana are published targets; ./swarm-otel-identity.nix - # covers them. - name = "a host running the collector, authelia, grafana and nats scrapes nats on loopback, authelia and grafana not"; + # All three are published targets; ./swarm-otel-identity.nix covers them. + name = "a host running the collector, authelia, grafana and nats scrapes none of them on loopback"; ok = let t = servicesHere.services.hyperhive.swarm.otel.scrapeTargets; - port = n: toString servicesHere.services.hyperhive.swarm.${n}.metricsPort; in - t.nats == "127.0.0.1:${port "nats"}" && !(t ? authelia) && !(t ? grafana); + servicesHere.services.hyperhive.deploy.nats.enable + && !(t ? authelia) + && !(t ? grafana) + && !(t ? nats); } { name = "hosting the swarm's shared services runs the forge"; diff --git a/nix/module-eval/nats-tls.nix b/nix/module-eval/nats-tls.nix index c0301447..b64d3695 100644 --- a/nix/module-eval/nats-tls.nix +++ b/nix/module-eval/nats-tls.nix @@ -362,6 +362,74 @@ let failedAssertions remoteLeafNoUrl == [ ] && !(remoteLeafNoUrl.systemd.services.hive-c0re.environment ? HIVE_C0RE_NATS_URL); } + { + # Only `/metrics`, from the exporter, behind the collector's subrequest: + # no `error_page` (a scraper would parse a login page as metrics), the + # monitoring endpoint never proxied, every other path refused. In + # self-signed mode the vhost names the queue's own leaf, as copied for + # nginx, not the swarm-services one. + name = "the queue's name serves only /metrics, behind the collector's auth subrequest, on the queue's own leaf"; + ok = + let + h = allLocal.services.hyperhive; + v = allLocal.services.nginx.virtualHosts.${natsName}; + locs = v.locations; + checksOf = l: lib.filter (lib.hasPrefix "auth_request") (lib.splitString "\n" l.extraConfig); + authz = locs."= /__nats_metrics_authz"; + in + lib.attrNames locs == [ + "/" + "= /__nats_metrics_authz" + "= /metrics" + ] + && locs."/".return == "403" + && locs."/".proxyPass == null + && locs."= /metrics".proxyPass == "http://127.0.0.1:${toString h.swarm.nats.metricsPort}/metrics" + && checksOf locs."= /metrics" == [ "auth_request /__nats_metrics_authz;" ] + && !(lib.hasInfix "error_page" locs."= /metrics".extraConfig) + && authz.proxyPass == "https://auth.t.local/api/authz/auth-request" + && lib.hasInfix "internal;" authz.extraConfig + && !(lib.any ( + l: + lib.hasInfix ":${toString h.swarm.nats.monitorPort}" ( + if l.proxyPass == null then "" else l.proxyPass + ) + ) (lib.attrValues locs)) + && v.forceSSL + && v.sslCertificate == "/var/lib/swarm-nats-gateway-tls/cert.pem" + && v.sslCertificateKey == "/var/lib/swarm-nats-gateway-tls/key.pem"; + } + { + # A host whose only swarm service is the queue: its gateway serves the + # queue's name with the queue's leaf, fronts no swarm-services name, and + # so neither waits for nor requires the swarm-services leaf. The copy + # unit is only wanted by nginx, never required, and a rotated queue leaf + # re-runs it. + name = "a host running only the queue renders its gateway without requiring the swarm-services leaf"; + ok = + let + h = queueNoStore.services.hyperhive; + s = queueNoStore.systemd.services; + copy = s.swarm-nats-gateway-cert; + in + h.deploy.nats.enable + && !h.deploy.hive-controller.enable + && h.gateway.enable + && h.gateway.useSelfSigned + && + queueNoStore.services.nginx.virtualHosts.${natsName}.sslCertificate + == "/var/lib/swarm-nats-gateway-tls/cert.pem" + && h.swarm.localServiceDomains == [ ] + && !(lib.elem natsName h.swarm.serviceDomains) + && !(lib.elem "hive-gateway-self-signed-cert.service" (s.swarm-services-cert.requiredBy or [ ])) + && !(lib.hasInfix "serves swarm names only with it" s.hive-gateway-self-signed-cert.script) + && lib.elem "nginx.service" copy.wantedBy + && lib.elem "nginx.service" copy.before + && !(lib.elem "nginx.service" (copy.requiredBy or [ ])) + && lib.hasInfix "throwaway self-signed" copy.script + && lib.hasInfix "chgrp nginx" copy.script + && lib.hasInfix "systemctl start swarm-nats-gateway-cert.service" s.swarm-bao-nats-tls.script; + } ]; in runGroup "nats-tls" cases diff --git a/nix/module-eval/swarm-otel-identity.nix b/nix/module-eval/swarm-otel-identity.nix index 04c0d047..2d27d63c 100644 --- a/nix/module-eval/swarm-otel-identity.nix +++ b/nix/module-eval/swarm-otel-identity.nix @@ -111,6 +111,7 @@ let grafanaMetricsUrl = "https://grafana.t.local/metrics"; logsMetricsUrl = "https://logs.t.local/metrics"; vmMetricsUrl = "https://metrics.t.local/metrics"; + natsMetricsUrl = "https://nats.t.local/metrics"; # The policy authelia applies to `subject` requesting `https://`, # where `path` is what authelia matches `resources` against: the cleaned path @@ -378,8 +379,9 @@ let grafanaMetricsUrl logsMetricsUrl vmMetricsUrl + natsMetricsUrl ] - && lib.length targets == 10 + && lib.length targets == 12 && lib.all (t: policy collector t == "one_factor") targets && lib.all (t: policy "user:someone" t == "deny") targets && lib.all (t: policy "group:admins" t == "deny") targets; @@ -427,7 +429,7 @@ let ]; matched = row: builtins.match re row.path != null; in - lib.length metricsRules == 8 + lib.length metricsRules == 10 && lib.all (r: r.resources == [ re ]) metricsRules && re == "^/metrics([?].*)?$" && lib.all (row: matched row == (row.nginx == "= /metrics")) table; @@ -621,6 +623,60 @@ let && lib.length jobs == 1 && (lib.head jobs).static_configs == [ { targets = [ "grafana.t.local" ]; } ]; } + { + name = "a collector on a host without the queue scrapes it by name"; + ok = + let + s = otelSettings otelRemoteAuthelia; + job = lib.findFirst (j: j.job_name == "nats") null s.receivers.prometheus.config.scrape_configs; + in + !otelRemoteAuthelia.services.hyperhive.deploy.nats.enable + && job != null + && job.static_configs == [ { targets = [ "nats.t.local" ]; } ] + && job.metrics_path == "/metrics" + && job.oauth2.endpoint_params.audience == natsMetricsUrl; + } + { + # The `deny` that confines the queue's `/metrics` is scoped to that path, + # and comes after the collector's rule. + name = "authelia confines the queue's /metrics to the collector"; + ok = + let + h = autheliaOnly.services.hyperhive; + collector = "oauth2:client:${h.swarm.otel.clientId}"; + rules = + autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules; + natsRules = lib.filter (r: r.domain == "nats.t.local") rules; + policy = subject: autheliaPolicy autheliaOnly subject "nats.t.local"; + in + map (r: r.policy) natsRules == [ + "one_factor" + "deny" + ] + && lib.all (r: r.resources == [ "^/metrics([?].*)?$" ]) natsRules + && (lib.head natsRules).subject == [ collector ] + && policy collector "/metrics" == "one_factor" + && policy "user:someone" "/metrics" == "deny" + && policy "user:someone" "/metrics?x=1" == "deny" + && policy "group:admins" "/metrics" == "deny"; + } + { + # Single host: the queue is scraped once, by its published name. + name = "a host running the queue and the collector scrapes it once, by name"; + ok = + let + o = allLocal.services.hyperhive.swarm.otel; + jobs = + lib.filter (j: j.job_name == "nats") + (otelSettings allLocal).receivers.prometheus.config.scrape_configs; + in + allLocal.services.hyperhive.deploy.nats.enable + && allLocal.services.hyperhive.deploy.swarm-otel.enable + && !(o.scrapeTargets ? nats) + && (o.publishedScrapeTargets.nats or null) == natsMetricsUrl + && lib.length jobs == 1 + && (lib.head jobs).static_configs == [ { targets = [ "nats.t.local" ]; } ]; + } { # Read against the roster the fixture declares rather than against # names spelled here: an arm naming `h1` passes on a single-hive