diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 7fa38086..26628c8b 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -172,18 +172,10 @@ Neither container is reachable except through the gateway: both bind loopback, and VictoriaMetrics authenticates nothing itself. The `metrics.` vhost forwards only the store's read API (the Prometheus query, series, label and metadata routes, plus the vmui web UI) -without a credential. It forwards the collector's OTLP push, and the -collector's scrape of the store's own `/metrics`, only for the swarm -collector's authelia client: authelia refuses any other session or client -there. It refuses every other path, the store's write and admin +without a credential. It forwards the collector's OTLP push only with an +authelia token, and refuses every other path, the store's write and admin API included, with 403. The bare domain redirects to vmui. -The collector scrapes the store's own metrics the same way, at -`https:///metrics`, so a collector on a host -that doesn't run the store still gets them. On this name the gateway admits -only the collector's client to `/metrics` and the push route; any other -session gets `deny`. - ### Logs (VictoriaLogs) Each hive ships its journals to one VictoriaLogs at `logs.`, diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index 6aaa3b53..a5c5d760 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -61,7 +61,7 @@ let collectorClientId = config.services.hyperhive.swarm.otel.clientId; collectorRegistered = lib.any (c: c.id == collectorClientId) cfg.oidc.clients; - # `target`, for the collector's client and nobody else. + # Every published metrics endpoint: the collector's client, and nobody else. # # The `deny` is the protection, and it has to be a rule of its own. A rule # with a `subject` does not match a subject outside it: authelia skips it and @@ -71,52 +71,38 @@ let # between a browser session and this data: `authn_strategies` on the authz # endpoint also accepts `CookieSession`, and a cookie carries no audience. # Until a collector is registered, the `deny` is the only rule. - collectorOnly = - target: - lib.optional collectorRegistered ( - target - // { - policy = "one_factor"; - subject = [ "oauth2:client:${collectorClientId}" ]; - } - ) - ++ [ (target // { policy = "deny"; }) ]; - - # The metrics store's name, every path. Only its push route and `/metrics` - # consult authelia (its read API is served with no check), and the - # collector is the one caller of both, so the rules take no `resources` - # and leave no path or query form to fall through to `default_policy`. - # ⚠️ An authenticated location added to that vhost for anyone else is - # refused here until it gets a rule ahead of the `deny`. - metricsStoreDomain = config.services.hyperhive.swarm.victoriametrics.domain; - metricsStoreRules = collectorOnly { domain = metricsStoreDomain; }; - + # # One pair per `swarm.otel.publishedScrapeTargets` entry, read off the same # URL the collector requests and its token is minted for, so a target # cannot be published without the rules that confine it to the collector. - # An entry on the metrics store's name is covered by `metricsStoreRules`. - metricsRules = lib.filter (r: r.domain != metricsStoreDomain) ( - lib.concatLists ( - lib.mapAttrsToList ( - job: url: - let - # The split `parsePublished` in ./swarm-otel.nix makes for the scrape - # job; the two must agree on what the URL's host and path are. - parts = builtins.match "https://([^/]+)(/.*)" url; - in - if parts == null then - throw "swarm.otel.publishedScrapeTargets.${job} = \"${url}\" is not an https URL with a path; authelia's access rule for it cannot be derived." - else - collectorOnly { - domain = builtins.elemAt parts 0; - # ⚠️ Authelia matches `resources` against the cleaned path PLUS the - # query string, while nginx's `= ` location ignores the query. - # A bare `^$` therefore misses `?x`, which the location - # still serves, and that request falls through to `default_policy`. - resources = [ "^${lib.escapeRegex (builtins.elemAt parts 1)}([?].*)?$" ]; + metricsRules = lib.concatLists ( + lib.mapAttrsToList ( + job: url: + let + # The split `parsePublished` in ./swarm-otel.nix makes for the scrape + # job; the two must agree on what the URL's host and path are. + parts = builtins.match "https://([^/]+)(/.*)" url; + target = { + domain = builtins.elemAt parts 0; + # ⚠️ Authelia matches `resources` against the cleaned path PLUS the + # query string, while nginx's `= ` location ignores the query. + # A bare `^$` therefore misses `?x`, which the location + # still serves, and that request falls through to `default_policy`. + resources = [ "^${lib.escapeRegex (builtins.elemAt parts 1)}([?].*)?$" ]; + }; + in + if parts == null then + throw "swarm.otel.publishedScrapeTargets.${job} = \"${url}\" is not an https URL with a path; authelia's access rule for it cannot be derived." + else + lib.optional collectorRegistered ( + target + // { + policy = "one_factor"; + subject = [ "oauth2:client:${collectorClientId}" ]; } - ) config.services.hyperhive.swarm.otel.publishedScrapeTargets - ) + ) + ++ [ (target // { policy = "deny"; }) ] + ) config.services.hyperhive.swarm.otel.publishedScrapeTargets ); # Upstream's `services.authelia.instances.` derives the unit, @@ -1152,10 +1138,9 @@ in # shadowed by a broader domain rule added later. rules = # The collector's client where one is registered, then - # `deny` — see `collectorOnly` above, which is where the + # `deny` — see `metricsRules` above, which is where the # reasoning for both halves lives. metricsRules - ++ metricsStoreRules # Also guarded on the domain being set: without it a null # apex would render a rule matching the string "null". ++ lib.optional (deployCfg.swarm-controller.enable && swarmDomain != null) { diff --git a/nix/host-modules/swarm-otel-service.nix b/nix/host-modules/swarm-otel-service.nix index 65483bdd..c63fe78b 100644 --- a/nix/host-modules/swarm-otel-service.nix +++ b/nix/host-modules/swarm-otel-service.nix @@ -358,7 +358,5 @@ in forgejo = "https://${hyperhiveCfg.swarm.forge.domain}/metrics"; # The vhost location is in ./swarm-authelia.nix. authelia = "https://${hyperhiveCfg.swarm.authelia.domain}/metrics"; - # The vhost location is in ./swarm-victoriametrics.nix. - victoriametrics = "https://${vmCfg.domain}/metrics"; }; } diff --git a/nix/host-modules/swarm-victoriametrics.nix b/nix/host-modules/swarm-victoriametrics.nix index ffd6e828..63164a2b 100644 --- a/nix/host-modules/swarm-victoriametrics.nix +++ b/nix/host-modules/swarm-victoriametrics.nix @@ -116,6 +116,22 @@ in services.hyperhive.gateway.enable = lib.mkDefault true; services.hyperhive.gateway.dns.enable = lib.mkDefault true; + # This store publishes its own health as prometheus metrics on the same + # listener it serves queries on, so the swarm's collector can scrape it + # with no exporter and no extra port. + # + # Declared here rather than in the collector's module because that is the + # rule the option carries: an entry exists only where the service that + # named it runs, which is what keeps scraper and target on one host by + # construction instead of by luck. + # + # The loopback literal is safe for this option and NOT for the write path, + # which is the distinction that matters now that the two differ. A scrape + # target is only ever read by a collector on this host, so loopback states + # a fact. The collector's push goes to the swarm name through the gateway, + # because the collector need not be here at all. + services.hyperhive.swarm.otel.scrapeTargets.victoriametrics = "127.0.0.1:${toString cfg.port}"; + services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // { listen = gatewayCfg.lib.listen; extraConfig = gatewayCfg.lib.securityHeaders; @@ -130,8 +146,8 @@ in # index page links mostly to paths refused above, so send it to vmui. "= /".return = "302 /vmui/"; - # The swarm collector's ingest route. `=` so it outranks the `/` - # prefix above. + # The swarm collector's ingest route, and the only authenticated thing + # on this vhost. `=` so it outranks the `/` prefix above. # # ⚠️ No `error_page 401 =302` here, and its absence is the point: a # redirect is right for a browser and wrong for a pusher, which would @@ -144,29 +160,10 @@ in ''; }; - # The store's own health, for the swarm collector wherever it runs: - # served on the query listener, so no exporter and no extra port. `=` - # for the same reason as the push route, and no `error_page` for the - # same reason: a scraper would parse a login page as metrics. - # - # The collector requests exactly this URL, declared as - # `publishedScrapeTargets.victoriametrics` in ./swarm-otel-service.nix; - # the two must agree, since that string is also the token's audience. - "= /metrics" = { - proxyPass = "http://127.0.0.1:${toString cfg.port}/metrics"; - extraConfig = '' - auth_request /__metrics_push_authz; - ''; - }; - - # The subrequest for both gated locations above. Authelia admits the - # collector's client anywhere on this name and refuses every other - # subject (`metricsStoreRules` in ./swarm-authelia.nix), so the scrape - # needs no subrequest of its own. Same target and header set as the - # sibling log store's, which took them from `swarm-ui.nix` — - # `X-Original-URL` and `X-Original-Method` are what authelia's - # auth-request implementation reads, and the address it compares the - # token's audience against. + # The subrequest. Same target and header set as the sibling log + # store's, which took them from `swarm-ui.nix` — `X-Original-URL` and + # `X-Original-Method` are what authelia's auth-request implementation + # reads, and the address it compares the token's audience against. "= /__metrics_push_authz" = { proxyPass = "https://${hyperhiveCfg.swarm.authelia.domain}/api/authz/auth-request"; # nixpkgs appends its OWN `Host $host` after extraConfig, which diff --git a/nix/module-eval/swarm-otel-identity.nix b/nix/module-eval/swarm-otel-identity.nix index ba6eb295..5303a992 100644 --- a/nix/module-eval/swarm-otel-identity.nix +++ b/nix/module-eval/swarm-otel-identity.nix @@ -106,7 +106,6 @@ let forgeMetricsUrl = "https://forge.t.local/metrics"; autheliaMetricsUrl = "https://auth.t.local/metrics"; - vmMetricsUrl = "https://metrics.t.local/metrics"; # The policy authelia applies to `subject` requesting `https://`, # where `path` is what authelia matches `resources` against: the cleaned path @@ -252,93 +251,6 @@ let && authz.proxyPass == "https://${h.swarm.authelia.domain}/api/authz/auth-request" && lib.hasInfix "internal;" authz.extraConfig; } - { - # The store's half of the same move, on a collector host that does not - # run the store. - name = "a collector on a host without the metrics store scrapes the store by name"; - ok = - let - s = otelSettings otelRemoteAuthelia; - job = lib.findFirst ( - j: j.job_name == "victoriametrics" - ) null s.receivers.prometheus.config.scrape_configs; - in - !otelRemoteAuthelia.services.hyperhive.deploy.victoriametrics.enable - && job != null - && job.static_configs == [ { targets = [ "metrics.t.local" ]; } ] - && job.metrics_path == "/metrics" - && job.oauth2.endpoint_params.audience == vmMetricsUrl; - } - { - # The metrics vhost is default-deny; `/metrics` reaches the store only - # through its own exact location, gated like the push route. The `/` - # clauses pin that the catch-all still refuses every other path. - name = "the metrics store's vhost serves /metrics only behind the collector's auth subrequest"; - ok = - let - h = allLocal.services.hyperhive; - locs = allLocal.services.nginx.virtualHosts.${h.swarm.victoriametrics.domain}.locations; - m = locs."= /metrics" or null; - in - m != null - && m.proxyPass == "http://127.0.0.1:${toString h.swarm.victoriametrics.port}/metrics" - && lib.hasInfix "auth_request /__metrics_push_authz;" m.extraConfig - && !(lib.hasInfix "error_page" m.extraConfig) - && lib.hasInfix "internal;" locs."= /__metrics_push_authz".extraConfig - && locs."/".return == "403" - && locs."/".proxyPass == null; - } - { - # The metrics store's name admits the collector's client on every path - # and refuses every other subject. The rules carry no `resources`, so no - # path or query spelling misses them and reaches `default_policy`, and - # they come before the apex rule. That is only right while the push - # route and `/metrics` are the vhost's sole authelia-gated locations, - # which the last clause pins. - name = "authelia admits only the collector anywhere on the metrics store's name"; - ok = - let - h = allLocal.services.hyperhive; - collector = "oauth2:client:${h.swarm.otel.clientId}"; - rules = - allLocal.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules; - indexed = lib.imap0 (i: r: r // { inherit i; }) rules; - storeRules = lib.filter (r: r.domain == "metrics.t.local") indexed; - apex = lib.findFirst (r: r.domain == "t.local") null indexed; - policy = subject: autheliaPolicy allLocal subject "metrics.t.local"; - paths = [ - "/opentelemetry/api/v1/push" - "/metrics" - "/metrics?x=1" - "/api/v1/query" - ]; - locs = allLocal.services.nginx.virtualHosts.${h.swarm.victoriametrics.domain}.locations; - in - map (r: removeAttrs r [ "i" ]) storeRules == [ - { - domain = "metrics.t.local"; - policy = "one_factor"; - subject = [ collector ]; - } - { - domain = "metrics.t.local"; - policy = "deny"; - } - ] - && apex != null - && lib.all (r: r.i < apex.i) storeRules - && lib.all (p: policy collector p == "one_factor") paths - && lib.all (p: policy "user:someone" p == "deny") paths - && lib.all (p: policy "group:admins" p == "deny") paths - && - lib.hasInfix "auth_request /__metrics_push_authz;" - locs."= /opentelemetry/api/v1/push".extraConfig - && - lib.attrNames (lib.filterAttrs (_: l: lib.hasInfix "auth_request" (l.extraConfig or "")) locs) == [ - "= /metrics" - "= /opentelemetry/api/v1/push" - ]; - } { # Every published target admits the collector's client and refuses any # other authenticated subject, with or without a query string. Read @@ -371,9 +283,8 @@ let lib.sort lib.lessThan (lib.attrValues h.swarm.otel.publishedScrapeTargets) == [ autheliaMetricsUrl forgeMetricsUrl - vmMetricsUrl ] - && lib.length targets == 6 + && lib.length targets == 4 && lib.all (t: policy collector t == "one_factor") targets && lib.all (t: policy "user:someone" t == "deny") targets && lib.all (t: policy "group:admins" t == "deny") targets; @@ -456,23 +367,6 @@ let && lib.length jobs == 1 && (lib.head jobs).static_configs == [ { targets = [ "auth.t.local" ]; } ]; } - { - # Same for the metrics store: one job, by its published name. - name = "a host running the metrics store and the collector scrapes the store once, by name"; - ok = - let - o = allLocal.services.hyperhive.swarm.otel; - jobs = - lib.filter (j: j.job_name == "victoriametrics") - (otelSettings allLocal).receivers.prometheus.config.scrape_configs; - in - allLocal.services.hyperhive.deploy.victoriametrics.enable - && allLocal.services.hyperhive.deploy.swarm-otel.enable - && !(o.scrapeTargets ? victoriametrics) - && (o.publishedScrapeTargets.victoriametrics or null) == vmMetricsUrl - && lib.length jobs == 1 - && (lib.head jobs).static_configs == [ { targets = [ "metrics.t.local" ]; } ]; - } { # The case above reads the options; this one reads the assertion they # feed, which is what refuses an operator's `nixos-rebuild`.