diff --git a/docs/networking/gateway.md b/docs/networking/gateway.md index 768df671..d0203c15 100644 --- a/docs/networking/gateway.md +++ b/docs/networking/gateway.md @@ -52,6 +52,8 @@ The gateway always terminates TLS. Which certificate it serves depends on what y `tls.certDir` together with `tls.acme.enable` fails an assertion — pick one. +⚠️ One vhost class has no TLS: a vhost bound to loopback for a local consumer. `grafana-metrics` (`nix/host-modules/swarm-grafana.nix`) is the one in the tree: it listens on `127.0.0.1` only and serves `= /metrics` from grafana's unix socket for this host's collector. The published `grafana.` vhost answers `/metrics` with 403. Everything with a routable name follows the table above. + ### ACME / Let's Encrypt (`tls.acme`) The simplest production path for a public domain: diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 154ac7a3..09c8f046 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -180,11 +180,6 @@ collector's authelia client: authelia refuses any other session or client there. It refuses every other path, the store's write and admin API included, with 403. The bare domain redirects to vmui. -The collector scrapes Grafana's own metrics at -`https:///metrics`, so a collector on a host without -Grafana still gets them. The gateway admits only the collector's client to -that path; any other session gets `deny`. - The collector scrapes the store's own metrics the same way, at `https:///metrics`, so a collector on a host that doesn't run the store still gets them. On this name the gateway admits diff --git a/nix/host-modules/glue-otel-scrape-targets.nix b/nix/host-modules/glue-otel-scrape-targets.nix index e01e8204..520864bf 100644 --- a/nix/host-modules/glue-otel-scrape-targets.nix +++ b/nix/host-modules/glue-otel-scrape-targets.nix @@ -1,20 +1,21 @@ -# Glue: declare the loopback scrape target of nats on the swarm's OTEL -# collector, wherever it shares a host with it. Authelia and grafana are -# `publishedScrapeTargets` jobs (./swarm-otel-service.nix); a loopback job of +# Glue: declare the loopback scrape targets of grafana and nats on the swarm's +# OTEL collector, wherever one of them shares a host with it. Authelia is a +# `publishedScrapeTargets` job (./swarm-otel-service.nix); a loopback job of # the same name fails ./swarm-otel.nix's job-name assertion. # -# ONE PAIRING — nats ← the collector. Deleting this leaves the collector not -# scraping nats: no error, no metrics. +# ONE FILE, TWO PAIRINGS — each service ← the collector, with the same shape. +# Deleting this leaves the collector scraping none of them: no error, no +# metrics. # -# The target is gated on the service AND the collector being enabled here. The +# Each target is gated on the service AND the collector being enabled here. The # loopback address is only honest when both run on this host; a target nobody -# can read asserts a collection that is not happening. Both default from -# `deploy.allSwarmServices`, but `mkDefault` is an invitation to override, so -# co-location is a property of the auto-deployed topology, not the modules. +# can read asserts a collection that is not happening. Both services default +# from `deploy.allSwarmServices`, but `mkDefault` is an invitation to override, +# so co-location is a property of the auto-deployed topology, not the modules. # # ⚠️ Silent when split: separate hosts are separate evaluations with no shared -# context, so a host running nats and no collector declares nothing and -# the collector's host never learns of it. No error, no warning — a +# context, so a host running a service and no collector declares nothing and +# the collector's host never learns of the service. No error, no warning — a # healthy exporter and an empty dashboard. Not guardable from here. # # ⚠️ `swarm.otel`, not `hyperhive.otel` — two collectors one word apart, and only @@ -32,6 +33,7 @@ let in { config.services.hyperhive.swarm.otel.scrapeTargets = { + grafana = withCollector "grafana" "127.0.0.1:${toString swarmCfg.grafana.metricsPort}"; nats = withCollector "nats" "127.0.0.1:${toString swarmCfg.nats.metricsPort}"; }; } diff --git a/nix/host-modules/swarm-grafana-service.nix b/nix/host-modules/swarm-grafana-service.nix index 4ccee30c..131c1cb0 100644 --- a/nix/host-modules/swarm-grafana-service.nix +++ b/nix/host-modules/swarm-grafana-service.nix @@ -1,7 +1,7 @@ -# The swarm's metrics UI as every hive sees it: the name it is served under -# and the OIDC client it is registered as, identical on every host. What the -# host running it decides, and the container itself, are in -# ./swarm-grafana.nix. +# The swarm's metrics UI as every hive sees it: the name it is served under, +# the port its `/metrics` is re-served on, and the OIDC client it is +# registered as, identical on every host. What the host running it decides, +# and the container itself, are in ./swarm-grafana.nix. { lib, config, @@ -20,11 +20,12 @@ in # `enable` is `services.hyperhive.deploy.grafana.enable` — see # ./deploy.nix. Whether this host runs the swarm's Grafana is a # deployment decision, and `swarm.*` has to be identical on every host. - # Here is what the service IS to every hive: the name it is served under - # and the OIDC client it is registered as. What the host running it - # decides — which build it runs, where its datasources point, which plugins - # are in its store path, and the socket directory it shares with nginx — is - # `deploy.grafana`, in ./swarm-grafana.nix. + # Here is what the service IS to every hive: the name it is served under, + # the port its `/metrics` is re-served on, and the OIDC client it is + # registered as. What the host running it decides — which build it runs, + # where its datasources point, which plugins are in its store path, and + # the socket directory it shares with nginx — is `deploy.grafana`, in + # ./swarm-grafana.nix. options.services.hyperhive.swarm.grafana = { machine = lib.mkOption { type = lib.types.str; @@ -50,13 +51,35 @@ in ⚠️ Changing this changes the OAuth redirect URI, which authelia matches exactly. Both sides move together because both derive from this option; an operator who pins one by hand breaks the login. + ''; + }; - The swarm collector scrapes Grafana's own metrics at - `https:///metrics`, from whichever host it runs on. - Grafana serves that endpoint to anyone, so the gateway puts it - behind authelia, whose rules admit only the collector's OAuth2 - client there: see `publishedScrapeTargets` in - ./swarm-otel-service.nix. + metricsPort = lib.mkOption { + type = lib.types.port; + default = 9095; + description = '' + Loopback port on which the gateway's nginx re-serves Grafana's + `/metrics`, and nothing else, so the swarm's collector can scrape it. + + ⚠️ **This is nginx's port, not Grafana's.** Grafana still claims none — + see {option}`services.hyperhive.deploy.grafana.socketDir` for why that + matters. A prometheus scrape target is a `host:port`, and it cannot + address a unix socket; rather than undo the socket decision, the one + endpoint a scraper needs gets a listener of its own. + + Bound to loopback and unauthenticated, which is the same posture every + other entry in + {option}`services.hyperhive.swarm.otel.scrapeTargets` has: those + targets are trusted by *proximity* rather than by credential. + Deliberately **not** the published `grafana.` vhost, which + would put an authorization decision in front of a scrape: that vhost + answers `/metrics` with 403, because Grafana serves it without + authentication. + + The number itself is arbitrary and free today; + `state/eval-port-collisions.sh` is what keeps it that way, since a + second claim on a port in this shared namespace produces no bind error + and nothing in any log. ''; }; diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index a8e30bf0..36fef4b0 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -20,7 +20,6 @@ let networkCfg = config.services.hyperhive.network; hyperhiveCfg = config.services.hyperhive; gatewayCfg = hyperhiveCfg.gateway; - autheliaDomain = hyperhiveCfg.swarm.authelia.domain; baoCfg = hyperhiveCfg.swarm.bao; baoDeploy = deployCfg.bao; @@ -460,43 +459,12 @@ in listen = gatewayCfg.lib.listen; extraConfig = gatewayCfg.lib.securityHeaders; # Grafana serves its Prometheus exposition to anyone, signed in or not - # (`metrics.enabled` below, no `basic_auth_*`), so this location's auth - # is the only gate on it. `=` is enough: nginx matches the decoded, - # normalised path, so `//metrics`, `/./metrics`, `/%6detrics` and - # `/metrics?x` all land here, and Grafana itself answers `/metrics/` with - # a login redirect. - # - # The collector requests exactly this URL, declared as - # `publishedScrapeTargets.grafana` in ./swarm-otel-service.nix; the two - # must agree, since that string is also the token's audience. Authelia's - # rules for it admit the collector's client and deny every other subject - # (`metricsRules` in ./swarm-authelia.nix). - locations."= /metrics" = { - proxyPass = "http://unix:${socketPath}:/metrics"; - extraConfig = '' - auth_request /__grafana_metrics_authz; - ''; - }; - # The subrequest, same shape as the forge's `= /__forge_metrics_authz` - # (./hive-forge/default.nix): no `error_page 401 =302`, so a scraper - # gets the 401 rather than a login page parsed as metrics. - locations."= /__grafana_metrics_authz" = { - proxyPass = "https://${autheliaDomain}/api/authz/auth-request"; - # nixpkgs appends its OWN `Host $host` after extraConfig, which would - # override verifiedProxyTo's — see hive-gateway/vhost-lib.nix. - recommendedProxySettings = false; - extraConfig = '' - internal; - ${gatewayCfg.lib.verifiedProxyTo autheliaDomain} - proxy_pass_request_body off; - proxy_set_header Content-Length ""; - proxy_set_header X-Original-Method $request_method; - proxy_set_header X-Original-URL $scheme://$http_host$request_uri; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Host $http_host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - ''; - }; + # (`metrics.enabled` below, no `basic_auth_*`), so the published name + # refuses it; the scrape goes through the loopback `grafana-metrics` + # vhost instead. `=` is enough: nginx matches the decoded, normalised + # path, so `//metrics`, `/./metrics`, `/%6detrics` and `/metrics?x` all + # land here, and Grafana itself answers `/metrics/` with a login redirect. + locations."= /metrics".return = "403"; locations."/" = { proxyPass = "http://unix:${socketPath}:/"; proxyWebsockets = true; @@ -512,6 +480,29 @@ in }; }; + # The scrape listener: Grafana's `/metrics` and nothing else, on loopback. + # + # No `serverName` and no TLS on purpose — this listener exists for exactly + # one client on the same host, so there is no name to match and nothing to + # verify. It is the only vhost here that is not reachable from outside the + # machine, and that is what lets it skip the authorization the published + # one carries. + # + # ⚠️ `= /metrics` is an EXACT match, not a prefix: it is the difference + # between exposing one endpoint and re-serving the whole of Grafana on an + # unauthenticated port. Widening it would be a silent authorization bypass, + # since nothing else about this block would have to change. The published + # vhost above refuses `/metrics`, so this listener is the only way to it. + services.nginx.virtualHosts."grafana-metrics" = { + listen = [ + { + addr = "127.0.0.1"; + port = cfg.metricsPort; + } + ]; + locations."= /metrics".proxyPass = "http://unix:${socketPath}:/metrics"; + }; + # Order the container after the host CA service so the bind source below # exists before nspawn sets the mount up. systemd.services."container@${cfg.machine}" = caTrust.containerOrdering; diff --git a/nix/host-modules/swarm-otel-service.nix b/nix/host-modules/swarm-otel-service.nix index 63b992fb..87ee68f1 100644 --- a/nix/host-modules/swarm-otel-service.nix +++ b/nix/host-modules/swarm-otel-service.nix @@ -362,7 +362,5 @@ in victoriametrics = "https://${vmCfg.domain}/metrics"; # The vhost location is in ./swarm-victorialogs.nix. victorialogs = "https://${hyperhiveCfg.swarm.victorialogs.domain}/metrics"; - # The vhost location is in ./swarm-grafana.nix. - grafana = "https://${hyperhiveCfg.swarm.grafana.domain}/metrics"; }; } diff --git a/nix/module-eval/forge-placement.nix b/nix/module-eval/forge-placement.nix index f04b5dcd..cb085e75 100644 --- a/nix/module-eval/forge-placement.nix +++ b/nix/module-eval/forge-placement.nix @@ -138,15 +138,16 @@ let && !(refusedOver bare "deploy.forgejo.sso.clientSecretFile"); } { - # Authelia and grafana are published targets; ./swarm-otel-identity.nix - # covers them. - name = "a host running the collector, authelia, grafana and nats scrapes nats on loopback, authelia and grafana not"; + # Authelia is a published target; ./swarm-otel-identity.nix covers it. + name = "a host running the collector, authelia, grafana and nats scrapes grafana and nats on loopback, authelia not"; ok = let t = servicesHere.services.hyperhive.swarm.otel.scrapeTargets; port = n: toString servicesHere.services.hyperhive.swarm.${n}.metricsPort; in - t.nats == "127.0.0.1:${port "nats"}" && !(t ? authelia) && !(t ? grafana); + t.grafana == "127.0.0.1:${port "grafana"}" + && t.nats == "127.0.0.1:${port "nats"}" + && !(t ? authelia); } { name = "hosting the swarm's shared services runs the forge"; diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix index d9cbeebf..61520d06 100644 --- a/nix/module-eval/grafana.nix +++ b/nix/module-eval/grafana.nix @@ -109,6 +109,12 @@ let && lib.hasInfix option a.message ) m.assertions; cases = [ + { + name = "grafana declares no scrape target on a host with no collector"; + ok = + !(grafanaOldPath.services.hyperhive.deploy.swarm-otel.enable) + && !(grafanaOldPath.services.hyperhive.swarm.otel.scrapeTargets ? grafana); + } { # The socket's directory is its access control, so it must stay dedicated # to this one socket. `readOnly` is what makes that inexpressible: a @@ -118,30 +124,22 @@ let ok = !(builtins.tryEval grafanaSocketDirSet.services.hyperhive.deploy.grafana.socketDir).success && - grafanaOldPath.services.nginx.virtualHosts.${grafanaOldPath.services.hyperhive.swarm.grafana.domain}.locations."= /metrics".proxyPass + grafanaOldPath.services.nginx.virtualHosts."grafana-metrics".locations."= /metrics".proxyPass == "http://unix:/run/swarm-grafana/grafana.sock:/metrics"; } { - # Grafana serves `/metrics` unauthenticated, so the published name gates - # it on the collector's auth subrequest: no `error_page 401 =302` (a - # scraper would parse a login page as metrics) and no `/__hive_authelia`. + # Grafana serves `/metrics` unauthenticated, so the published name must + # answer it itself; the scrape reads the loopback vhost above instead. # `=` is what catches the normalised variants and still leaves the rest # of Grafana proxied. - name = "grafana's published vhost serves /metrics only behind the collector's auth subrequest"; + name = "grafana's published vhost refuses /metrics and still proxies everything else"; ok = let g = grafanaOldPath.services.hyperhive.swarm.grafana; locs = grafanaOldPath.services.nginx.virtualHosts.${g.domain}.locations; - checksOf = l: lib.filter (lib.hasPrefix "auth_request") (lib.splitString "\n" l.extraConfig); - authz = locs."= /__grafana_metrics_authz"; in - locs ? "= /metrics" - && locs."= /metrics".return == null - && checksOf locs."= /metrics" == [ "auth_request /__grafana_metrics_authz;" ] - && !(lib.hasInfix "error_page" locs."= /metrics".extraConfig) - && authz.proxyPass == "https://auth.t.local/api/authz/auth-request" - && lib.hasInfix "internal;" authz.extraConfig - && !(grafanaOldPath.services.nginx.virtualHosts ? "grafana-metrics") + locs."= /metrics".return == "403" + && locs."= /metrics".proxyPass == null && lib.hasPrefix "http://unix:" locs."/".proxyPass; } { diff --git a/nix/module-eval/swarm-otel-identity.nix b/nix/module-eval/swarm-otel-identity.nix index 04c0d047..eb2b49c4 100644 --- a/nix/module-eval/swarm-otel-identity.nix +++ b/nix/module-eval/swarm-otel-identity.nix @@ -108,7 +108,6 @@ let forgeMetricsUrl = "https://forge.t.local/metrics"; autheliaMetricsUrl = "https://auth.t.local/metrics"; - grafanaMetricsUrl = "https://grafana.t.local/metrics"; logsMetricsUrl = "https://logs.t.local/metrics"; vmMetricsUrl = "https://metrics.t.local/metrics"; @@ -375,11 +374,10 @@ let lib.sort lib.lessThan (lib.attrValues h.swarm.otel.publishedScrapeTargets) == [ autheliaMetricsUrl forgeMetricsUrl - grafanaMetricsUrl logsMetricsUrl vmMetricsUrl ] - && lib.length targets == 10 + && lib.length targets == 8 && lib.all (t: policy collector t == "one_factor") targets && lib.all (t: policy "user:someone" t == "deny") targets && lib.all (t: policy "group:admins" t == "deny") targets; @@ -427,7 +425,7 @@ let ]; matched = row: builtins.match re row.path != null; in - lib.length metricsRules == 8 + lib.length metricsRules == 6 && lib.all (r: r.resources == [ re ]) metricsRules && re == "^/metrics([?].*)?$" && lib.all (row: matched row == (row.nginx == "= /metrics")) table; @@ -564,63 +562,6 @@ let && lib.length jobs == 1 && (lib.head jobs).static_configs == [ { targets = [ "logs.t.local" ]; } ]; } - { - name = "a collector on a host without grafana scrapes it by name"; - ok = - let - s = otelSettings otelRemoteAuthelia; - job = lib.findFirst (j: j.job_name == "grafana") null s.receivers.prometheus.config.scrape_configs; - in - !otelRemoteAuthelia.services.hyperhive.deploy.grafana.enable - && job != null - && job.static_configs == [ { targets = [ "grafana.t.local" ]; } ] - && job.metrics_path == "/metrics" - && job.oauth2.endpoint_params.audience == grafanaMetricsUrl; - } - { - # The `deny` that confines grafana's `/metrics` is scoped to that path, - # and comes after the collector's rule: every other path on the name - # still reaches `default_policy`, as it did with no rule at all. - name = "authelia confines grafana's /metrics to the collector and leaves its other paths alone"; - ok = - let - h = autheliaOnly.services.hyperhive; - collector = "oauth2:client:${h.swarm.otel.clientId}"; - rules = - autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules; - grafanaRules = lib.filter (r: r.domain == "grafana.t.local") rules; - policy = subject: autheliaPolicy autheliaOnly subject "grafana.t.local"; - in - map (r: r.policy) grafanaRules == [ - "one_factor" - "deny" - ] - && lib.all (r: r.resources == [ "^/metrics([?].*)?$" ]) grafanaRules - && (lib.head grafanaRules).subject == [ collector ] - && policy collector "/metrics" == "one_factor" - && policy "user:someone" "/metrics" == "deny" - && policy "user:someone" "/metrics?x=1" == "deny" - && policy "group:admins" "/metrics" == "deny" - && policy "user:someone" "/" == "one_factor" - && policy "user:someone" "/login/generic_oauth" == "one_factor"; - } - { - # Single host: grafana is scraped once, by its published name. - name = "a host running grafana and the collector scrapes it once, by name"; - ok = - let - o = allLocal.services.hyperhive.swarm.otel; - jobs = - lib.filter (j: j.job_name == "grafana") - (otelSettings allLocal).receivers.prometheus.config.scrape_configs; - in - allLocal.services.hyperhive.deploy.grafana.enable - && allLocal.services.hyperhive.deploy.swarm-otel.enable - && !(o.scrapeTargets ? grafana) - && (o.publishedScrapeTargets.grafana or null) == grafanaMetricsUrl - && lib.length jobs == 1 - && (lib.head jobs).static_configs == [ { targets = [ "grafana.t.local" ]; } ]; - } { # Read against the roster the fixture declares rather than against # names spelled here: an arm naming `h1` passes on a single-hive