diff --git a/nix/host-modules/swarm-otel-service.nix b/nix/host-modules/swarm-otel-service.nix index 65483bdd..87ee68f1 100644 --- a/nix/host-modules/swarm-otel-service.nix +++ b/nix/host-modules/swarm-otel-service.nix @@ -360,5 +360,7 @@ in authelia = "https://${hyperhiveCfg.swarm.authelia.domain}/metrics"; # The vhost location is in ./swarm-victoriametrics.nix. victoriametrics = "https://${vmCfg.domain}/metrics"; + # The vhost location is in ./swarm-victorialogs.nix. + victorialogs = "https://${hyperhiveCfg.swarm.victorialogs.domain}/metrics"; }; } diff --git a/nix/host-modules/swarm-victorialogs.nix b/nix/host-modules/swarm-victorialogs.nix index 6d9e4f82..b1b946b4 100644 --- a/nix/host-modules/swarm-victorialogs.nix +++ b/nix/host-modules/swarm-victorialogs.nix @@ -68,8 +68,8 @@ let }; baoCfg = hyperhiveCfg.swarm.bao; - # Shared host netns, like every sibling swarm container: the collector - # and Grafana reach this at 127.0.0.1:. + # Shared host netns, like every sibling swarm container: Grafana reaches + # this at 127.0.0.1:. privateNetwork = false; in { @@ -215,18 +215,6 @@ in limit_req_zone $binary_remote_addr zone=swarm_logs_agent_auth:1m rate=${deployCfg.victorialogs.authRate}; ''; - # This store publishes its own health as prometheus metrics on the same - # listener it serves queries on, so the swarm's collector scrapes it with - # no exporter and no extra port — same arrangement as the metrics store. - # - # Declared here rather than in the collector's module because that is the - # rule the option carries: an entry exists only where the service that - # named it runs. ⚠️ That constrains the TARGET, not the scraper — a - # deployment that splits this container away from the collector's host - # silently drops the entry, and no assertion can see it, because separate - # hosts are separate evaluations. - services.hyperhive.swarm.otel.scrapeTargets.victorialogs = "127.0.0.1:${toString cfg.port}"; - # The gateway name, inside `deployCfg.victorialogs.enable` — same # "only the host that runs the service may claim the name" guard every # sibling swarm-service module uses (`swarm-grafana.nix`, @@ -313,6 +301,23 @@ in ''; }; + # The store's own health, which it publishes as prometheus metrics + # on the listener it serves queries on. The swarm collector requests + # exactly this URL, declared as `publishedScrapeTargets.victorialogs` + # in ./swarm-otel-service.nix; the two must agree, since that string + # is also the token's audience. + # + # Bare `auth_request` to authelia, as on the ingest route above, and + # never `/__swarm_logs_auth`, which admits an agent's bearer: + # authelia's rules for this URL admit the collector's client and + # deny every other subject (see `metricsRules` in ./swarm-authelia.nix). + "= /metrics" = { + proxyPass = "http://127.0.0.1:${toString cfg.port}/metrics"; + extraConfig = '' + auth_request /__hive_authelia; + ''; + }; + # `rewrite … last` re-matches locations, so the subrequest lands in # one of the two internal locations named by the map. "= /__swarm_logs_auth" = { diff --git a/nix/module-eval/swarm-otel-identity.nix b/nix/module-eval/swarm-otel-identity.nix index ba6eb295..0172ff85 100644 --- a/nix/module-eval/swarm-otel-identity.nix +++ b/nix/module-eval/swarm-otel-identity.nix @@ -106,6 +106,7 @@ let forgeMetricsUrl = "https://forge.t.local/metrics"; autheliaMetricsUrl = "https://auth.t.local/metrics"; + logsMetricsUrl = "https://logs.t.local/metrics"; vmMetricsUrl = "https://metrics.t.local/metrics"; # The policy authelia applies to `subject` requesting `https://`, @@ -371,9 +372,10 @@ let lib.sort lib.lessThan (lib.attrValues h.swarm.otel.publishedScrapeTargets) == [ autheliaMetricsUrl forgeMetricsUrl + logsMetricsUrl vmMetricsUrl ] - && lib.length targets == 6 + && lib.length targets == 8 && lib.all (t: policy collector t == "one_factor") targets && lib.all (t: policy "user:someone" t == "deny") targets && lib.all (t: policy "group:admins" t == "deny") targets; @@ -421,7 +423,7 @@ let ]; matched = row: builtins.match re row.path != null; in - lib.length metricsRules == 4 + lib.length metricsRules == 6 && lib.all (r: r.resources == [ re ]) metricsRules && re == "^/metrics([?].*)?$" && lib.all (row: matched row == (row.nginx == "= /metrics")) table; @@ -479,6 +481,85 @@ let name = "a single-host swarm passes the loopback-vs-published job name assertion"; ok = !(refusedOver allLocal "is declared as both a loopback"); } + { + name = "a collector on a host without the log store scrapes it by name"; + ok = + let + s = otelSettings otelRemoteAuthelia; + job = lib.findFirst ( + j: j.job_name == "victorialogs" + ) null s.receivers.prometheus.config.scrape_configs; + in + !otelRemoteAuthelia.services.hyperhive.deploy.victorialogs.enable + && job != null + && job.static_configs == [ { targets = [ "logs.t.local" ]; } ] + && job.metrics_path == "/metrics" + && job.oauth2.endpoint_params.audience == logsMetricsUrl; + } + { + # `= /metrics` must carry the bare authelia check and nothing else: the + # agents' `/__swarm_logs_auth` would admit an agent's bearer, and the + # browser block's `error_page 401 =302` hands a scraper a login page. + # The other routes keep their own checks. + name = "the log store's vhost serves /metrics behind authelia alone, its other routes unchanged"; + ok = + let + h = allLocal.services.hyperhive; + locs = allLocal.services.nginx.virtualHosts.${h.swarm.victorialogs.domain}.locations; + checksOf = l: lib.filter (lib.hasPrefix "auth_request") (lib.splitString "\n" l.extraConfig); + in + h.deploy.victorialogs.enable + && locs ? "= /metrics" + && locs."= /metrics".proxyPass == "http://127.0.0.1:${toString h.swarm.victorialogs.port}/metrics" + && checksOf locs."= /metrics" == [ "auth_request /__hive_authelia;" ] + && !(lib.hasInfix "error_page" locs."= /metrics".extraConfig) + && lib.hasInfix "error_page 401 =302" locs."/".extraConfig + && lib.elem "auth_request /__hive_authelia;" (checksOf locs."/") + && checksOf locs."^~ /select/logsql/" == [ "auth_request /__swarm_logs_auth;" ]; + } + { + # The `deny` that confines the log store's `/metrics` is scoped to that + # path, and comes after the collector's rule: every other path on the + # name still reaches `default_policy`, as it did with no rule at all. + name = "authelia confines the log store's /metrics to the collector and leaves its other paths alone"; + ok = + let + h = autheliaOnly.services.hyperhive; + collector = "oauth2:client:${h.swarm.otel.clientId}"; + rules = + autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules; + logsRules = lib.filter (r: r.domain == "logs.t.local") rules; + policy = subject: autheliaPolicy autheliaOnly subject "logs.t.local"; + in + map (r: r.policy) logsRules == [ + "one_factor" + "deny" + ] + && lib.all (r: r.resources == [ "^/metrics([?].*)?$" ]) logsRules + && (lib.head logsRules).subject == [ collector ] + && policy collector "/metrics" == "one_factor" + && policy "user:someone" "/metrics" == "deny" + && policy "user:someone" "/metrics?x=1" == "deny" + && policy "user:someone" "/" == "one_factor" + && policy "user:someone" "/select/logsql/query" == "one_factor" + && policy "user:someone" "/insert/opentelemetry/v1/logs" == "one_factor"; + } + { + # Single host: the store is scraped once, by its published name. + name = "a host running the log store and the collector scrapes it once, by name"; + ok = + let + o = allLocal.services.hyperhive.swarm.otel; + jobs = + lib.filter (j: j.job_name == "victorialogs") + (otelSettings allLocal).receivers.prometheus.config.scrape_configs; + in + allLocal.services.hyperhive.deploy.victorialogs.enable + && allLocal.services.hyperhive.deploy.swarm-otel.enable + && !(o.scrapeTargets ? victorialogs) + && lib.length jobs == 1 + && (lib.head jobs).static_configs == [ { targets = [ "logs.t.local" ]; } ]; + } { # Read against the roster the fixture declares rather than against # names spelled here: an arm naming `h1` passes on a single-hive