# `checks.module-eval-swarm-otel-identity` — see ./lib.nix for the shared # rationale (why this suite exists, naming convention, "evaluates # not executes"). { pkgs, lib, self, nixosSystem, shared, }: let inherit (import ./lib.nix { inherit pkgs lib self nixosSystem ; }) hive runGroup otelSettings autheliaRules autheliaSubjectRulesClosed ; # A collector holding no store identity at all. Unlike Grafana's mirror # image, this is not a refused shape: the collector still receives every # hive's telemetry with nothing to push authenticated with, which is the # already-supported degrade `haveCollectorSecret` names above the module's # `let`. What this fixture is for is checking the reading unit itself does # not render, rather than rendering with an env var nothing filled in. otelNoIdentity = hive { deploy.swarm-otel.enable = true; swarm.authelia.url = "https://auth.example.invalid"; }; # 🩸 The shape the degrade above must not swallow: a collector on a host that # demonstrably READS the store — it holds the hive's own leaf — and is missing # only the collector's own pair. Before the four-way split this host rendered # the reading unit off `clientCertFile` alone, so the silence it gets now is a # regression rather than the supported degrade `otelNoIdentity` stands for. The # two fixtures differ in exactly that one pair, which is what lets the arms # below separate a refusal from a degrade. inherit (shared) otelReaderMissingLeaf; # The same deployment named in full, which must stay buildable. Pairs with the # fixture above so the refusal is pinned to the omission and not to reading a # remote store at all. inherit (shared) otelReaderNamedInFull; # Did the module refuse this host, and over which option. An assertion is a # config VALUE until something forces it — `.config` never throws — so a # fixture in a refused state stays evaluable and the refusal reads back as # data. Matched on the option name the message names rather than on its prose: # the option name is the part an operator has to act on, and a message that # stopped naming it would be the actual defect. refusedOver = m: option: lib.any (a: !a.assertion && lib.hasInfix option a.message) m.assertions; # authelia somewhere else, the credential delivered by hand. Whether this # collector authenticates must follow the credential, never another # service's placement. otelRemoteAuthelia = hive { deploy.swarm-otel.enable = true; deploy.authelia.enable = false; # Where that elsewhere IS. Running no IdP does not mean knowing no IdP: # the authenticator this fixture exists to render puts this address in its # `token_url`, so a hive with a secret and no URL has a credential it can # present nowhere. swarm.authelia.url = "https://auth.example.invalid"; deploy.swarm-otel.clientSecretFile = "/var/lib/swarm-otel-oidc/by-hand.secret"; }; # A collector whose ONLY scrape work is published: loopback targets forced # empty, published jobs only. Unreachable in a real deploy today — # the module seeds `scrapeTargets.collector` under its own `enable`, so the # loopback set is never empty on its own — which is exactly why the arm # below needs a fixture that takes that seeding away. `mkForce` is what # does it, and it leaves the collector itself enabled: the state under test # is a running collector with no self-scrape, not an absent one. otelOnlyPublished = hive { deploy.swarm-otel.enable = true; swarm.otel.scrapeTargets = lib.mkForce { }; swarm.otel.publishedScrapeTargets.remote = "https://remote.t.local/metrics"; }; # Two hives in the roster, which no other fixture here has: every one of # them declares `swarm.hives.h1` alone, so a per-hive arm written against # one of those passes on a hardcoded literal. otelTwoHives = hive { deploy.swarm-otel.enable = true; deploy.authelia.enable = true; swarm.hives.h2.domain = "h2.t.local"; }; # authelia with neither the forge nor the collector beside it. inherit (shared) autheliaOnly; # Every swarm service on one host, the collector and authelia included. inherit (shared) allLocal; forgeMetricsUrl = "https://forge.t.local/metrics"; autheliaMetricsUrl = "https://auth.t.local/metrics"; grafanaMetricsUrl = "https://grafana.t.local/metrics"; logsMetricsUrl = "https://logs.t.local/metrics"; vmMetricsUrl = "https://metrics.t.local/metrics"; natsMetricsUrl = "https://nats.t.local/metrics"; # The policy authelia applies to `subject` requesting `https://`, # where `path` is what authelia matches `resources` against: the cleaned path # plus `?` when there is one. # It is the first rule whose domain, resources and subject all match, else # `default_policy`. ⚠️ A rule with a `subject` list does NOT match a subject # outside it — authelia moves on to the next rule — which is the semantics a # check of the rule's own fields cannot see. autheliaPolicy = m: subject: domain: path: let ac = m.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control; matches = r: r.domain == domain && lib.any (re: builtins.match re path != null) (r.resources or [ ".*" ]) && (!(r ? subject) || builtins.elem subject r.subject); hit = lib.findFirst matches null ac.rules; in if hit == null then ac.default_policy else hit.policy; cases = [ { # The collector's non-assertion, the deliberate mirror of Grafana's # assertion two cases up: a host with no store identity is a supported, # merely degraded shape here, so the reading unit simply does not exist # rather than refusing the build. `haveCollectorSecret` is what the # degrade already reads, unchanged by this slice. name = "a collector with no store identity renders no reading unit, and is not refused"; ok = !(otelNoIdentity.systemd.services ? swarm-bao-otel-oidc) && otelNoIdentity.services.hyperhive.deploy.swarm-otel.clientSecretFile == null && !(lib.any (a: !a.assertion) otelNoIdentity.assertions); } { # The collector's half of the same split, and a different arm from the # authenticator case below: this one reads the PATH the unit loads, so a # reader left on a source that is non-null but wrong still fails. name = "a by-hand otel secret path is the one the collector loads as a credential"; ok = lib.any (c: lib.hasInfix "/var/lib/swarm-otel-oidc/by-hand.secret" c) otelRemoteAuthelia.containers.swarm-otel.config.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential; } { # The collector authenticates because it HOLDS a credential, not because # authelia happens to share its host. Gating on the other service's # placement renders a collector that pushes unauthenticated wherever # authelia lives elsewhere — one of the supported shapes. name = "a collector with a hand-delivered secret authenticates without authelia beside it"; ok = let s = otelSettings otelRemoteAuthelia; in (s.exporters."otlphttp/victoriametrics" ? auth) && builtins.elem "oauth2client/victoriametrics" s.service.extensions; } { # Defining a receiver and attaching it are two separate lists, and the # two gates were spelled differently: the receiver appeared for either # scrape option, the pipeline only for the loopback one. A published- # only collector therefore rendered scrape configs that reached no # pipeline — requested, parsed, delivered nowhere, and valid enough to # deploy. The receiver clause is what stops the arm passing for the # wrong reason, by an empty `prometheus` never rendering at all. name = "a published-only collector attaches its prometheus receiver to the swarm pipeline"; ok = let s = otelSettings otelOnlyPublished; in otelOnlyPublished.services.hyperhive.swarm.otel.scrapeTargets == { } && otelOnlyPublished.services.hyperhive.swarm.otel.publishedScrapeTargets != { } && (s.receivers ? prometheus) && builtins.elem "prometheus" s.service.pipelines."metrics/swarm".receivers; } { # The collector reads its own host's target list, so a forge entry # declared only where the forge runs is a job no split deploy renders. # `otelRemoteAuthelia` runs neither the forge nor authelia. name = "a collector on a host without the forge scrapes the forge by name"; ok = let s = otelSettings otelRemoteAuthelia; job = lib.findFirst (j: j.job_name == "forgejo") null s.receivers.prometheus.config.scrape_configs; in !otelRemoteAuthelia.services.hyperhive.deploy.forgejo.enable && job != null && job.static_configs == [ { targets = [ "forge.t.local" ]; } ] && job.metrics_path == "/metrics" && job.oauth2.endpoint_params.audience == forgeMetricsUrl && builtins.elem "prometheus" s.service.pipelines."metrics/swarm".receivers; } { # The other half: authelia refuses a token for an audience its client # was not registered for, and registers it from its OWN host's list. name = "authelia on a host without the forge or the collector admits the collector's forge scrape"; ok = let h = autheliaOnly.services.hyperhive; client = lib.findFirst (c: c.id == h.swarm.otel.clientId) null h.swarm.authelia.oidc.clients; rules = autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules; in !h.deploy.forgejo.enable && !h.deploy.swarm-otel.enable && client != null && builtins.elem forgeMetricsUrl client.audience && builtins.elem { domain = "forge.t.local"; resources = [ "^/metrics([?].*)?$" ]; policy = "one_factor"; subject = [ "oauth2:client:${h.swarm.otel.clientId}" ]; } rules; } { # Authelia's half of the same move: its target is published by name, so # a collector anywhere in the swarm has a job for it. name = "a collector on a host without authelia scrapes authelia by name"; ok = let s = otelSettings otelRemoteAuthelia; job = lib.findFirst (j: j.job_name == "authelia") null s.receivers.prometheus.config.scrape_configs; in !otelRemoteAuthelia.services.hyperhive.deploy.authelia.enable && job != null && job.static_configs == [ { targets = [ "auth.t.local" ]; } ] && job.metrics_path == "/metrics" && job.oauth2.endpoint_params.audience == autheliaMetricsUrl; } { # The location the collector's job requests: gated, and proxying to the # loopback listener that has no authentication of its own. name = "authelia's vhost serves /metrics only behind the collector's auth subrequest"; ok = let h = autheliaOnly.services.hyperhive; locs = autheliaOnly.services.nginx.virtualHosts.${h.swarm.authelia.domain}.locations; authz = locs."= /__authelia_metrics_authz"; in locs."= /metrics".proxyPass == "http://127.0.0.1:${toString h.swarm.authelia.metricsPort}/metrics" && lib.hasInfix "auth_request /__authelia_metrics_authz;" locs."= /metrics".extraConfig && authz.proxyPass == "https://${h.swarm.authelia.domain}/api/authz/auth-request" && lib.hasInfix "internal;" authz.extraConfig; } { # The store's half of the same move, on a collector host that does not # run the store. name = "a collector on a host without the metrics store scrapes the store by name"; ok = let s = otelSettings otelRemoteAuthelia; job = lib.findFirst ( j: j.job_name == "victoriametrics" ) null s.receivers.prometheus.config.scrape_configs; in !otelRemoteAuthelia.services.hyperhive.deploy.victoriametrics.enable && job != null && job.static_configs == [ { targets = [ "metrics.t.local" ]; } ] && job.metrics_path == "/metrics" && job.oauth2.endpoint_params.audience == vmMetricsUrl; } { # The metrics vhost is default-deny; `/metrics` reaches the store only # through its own exact location, gated like the push route. The `/` # clauses pin that the catch-all still refuses every other path. name = "the metrics store's vhost serves /metrics only behind the collector's auth subrequest"; ok = let h = allLocal.services.hyperhive; locs = allLocal.services.nginx.virtualHosts.${h.swarm.victoriametrics.domain}.locations; m = locs."= /metrics" or null; in m != null && m.proxyPass == "http://unix:/run/swarm-victoriametrics/victoriametrics.sock:/metrics" && lib.hasInfix "auth_request /__metrics_authelia;" m.extraConfig && !(lib.hasInfix "error_page" m.extraConfig) && lib.hasInfix "internal;" locs."= /__metrics_authelia".extraConfig && locs."/".return == "403" && locs."/".proxyPass == null; } { # The metrics store's name admits the collector's client on every path, # an operator on every path but the collector's two, and nobody else. # Only the collector's two paths carry `resources`, as prefixes, so no # path or query spelling of them reaches the operator rule, and no other # spelling reaches `default_policy`. The rules come before the apex # rule. The collector's two locations call authelia directly, so a # bearer there never reaches Grafana's check. name = "authelia admits the collector anywhere on the metrics store's name and operators only off its two paths"; ok = let h = allLocal.services.hyperhive; collector = "oauth2:client:${h.swarm.otel.clientId}"; rules = allLocal.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules; indexed = lib.imap0 (i: r: r // { inherit i; }) rules; storeRules = lib.filter (r: r.domain == "metrics.t.local") indexed; apex = lib.findFirst (r: r.domain == "t.local") null indexed; policy = subject: autheliaPolicy allLocal subject "metrics.t.local"; collectorPaths = [ "/opentelemetry/api/v1/push" "/opentelemetry/api/v1/push?x=1" "/metrics" "/metrics?x=1" ]; readPaths = [ "/api/v1/query" "/api/v1/query_range?query=up" "/vmui/" ]; locs = allLocal.services.nginx.virtualHosts.${h.swarm.victoriametrics.domain}.locations; in lib.length storeRules == 4 && removeAttrs (lib.last storeRules) [ "i" ] == { domain = "metrics.t.local"; policy = "deny"; } && apex != null && lib.all (r: r.i < apex.i) storeRules && lib.all (p: policy collector p == "one_factor") (collectorPaths ++ readPaths) && lib.all (p: policy "user:someone" p == "deny") (collectorPaths ++ readPaths) && lib.all (p: policy "group:admins" p == "deny") collectorPaths && lib.all (p: policy "group:admins" p == "one_factor") readPaths && lib.hasInfix "auth_request /__metrics_authelia;" locs."= /opentelemetry/api/v1/push".extraConfig && lib.hasInfix "auth_request /__metrics_authelia;" locs."= /metrics".extraConfig; } { # Every published target admits the collector's client and refuses any # other authenticated subject, with or without a query string. Read # through authelia's first-match semantics: a client-scoped rule alone # lets an operator's session fall through to `default_policy = one_factor`, # and so does a request the rule's regex does not match. name = "every published metrics URL admits the collector and refuses a plain session"; ok = let h = autheliaOnly.services.hyperhive; collector = "oauth2:client:${h.swarm.otel.clientId}"; targets = lib.concatMap ( url: let t = builtins.match "https://([^/]+)(/.*)" url; in map (q: [ (builtins.elemAt t 0) "${builtins.elemAt t 1}${q}" ]) [ "" "?x=1" ] ) (lib.attrValues h.swarm.otel.publishedScrapeTargets); policy = subject: t: autheliaPolicy autheliaOnly subject (builtins.elemAt t 0) (builtins.elemAt t 1); in lib.sort lib.lessThan (lib.attrValues h.swarm.otel.publishedScrapeTargets) == [ autheliaMetricsUrl forgeMetricsUrl grafanaMetricsUrl logsMetricsUrl vmMetricsUrl natsMetricsUrl ] && lib.length targets == 12 && lib.all (t: policy collector t == "one_factor") targets && lib.all (t: policy "user:someone" t == "deny") targets && lib.all (t: policy "group:admins" t == "deny") targets; } { # Which request forms reach `= /metrics`, and whether the rule's regex # matches the path authelia derives from each. `nginx` records the # location each form lands in (measured against a stand-alone nginx); # `path` is authelia's cleaned path plus query for that form. The # invariant: every form the gated location serves is matched by the rule, # so none of them reaches `default_policy`. name = "the metrics rule matches every request form the gated location serves"; ok = let rules = autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules; # The metrics store's own name is the case above's; its rules carry # `resources` for its push route and `/metrics` together. metricsRules = lib.filter (r: r ? resources && r.domain != "metrics.t.local") rules; re = lib.head (lib.head metricsRules).resources; table = [ { request = "/metrics"; nginx = "= /metrics"; path = "/metrics"; } { request = "/metrics?x=1"; nginx = "= /metrics"; path = "/metrics?x=1"; } { request = "//metrics"; nginx = "= /metrics"; path = "/metrics"; } { request = "/metrics/"; nginx = "= /metrics/ (forge: 404) or /"; path = "/metrics/"; } { request = "/metricsx"; nginx = "/"; path = "/metricsx"; } ]; matched = row: builtins.match re row.path != null; in lib.length metricsRules == 10 && lib.all (r: r.resources == [ re ]) metricsRules && re == "^/metrics([?].*)?$" && lib.all (row: matched row == (row.nginx == "= /metrics")) table; } { # Forgejo strips trailing slashes before routing, so `/metrics/` reaches # its metrics handler unless nginx refuses it first. name = "the forge refuses /metrics/ rather than proxying it to forgejo's metrics"; ok = let locs = allLocal.services.nginx.virtualHosts.${allLocal.services.hyperhive.swarm.forge.domain}.locations; in (locs."= /metrics/".return or null) == "404"; } { # Single host: authelia is scraped once, by its published name. A # loopback job beside it would scrape the same listener twice — and share # its job name, which ./swarm-otel.nix refuses. name = "a host running authelia and the collector scrapes authelia once, by name"; ok = let o = allLocal.services.hyperhive.swarm.otel; jobs = lib.filter (j: j.job_name == "authelia") (otelSettings allLocal).receivers.prometheus.config.scrape_configs; in allLocal.services.hyperhive.deploy.authelia.enable && allLocal.services.hyperhive.deploy.swarm-otel.enable && !(o.scrapeTargets ? authelia) && (o.publishedScrapeTargets.authelia or null) == autheliaMetricsUrl && lib.length jobs == 1 && (lib.head jobs).static_configs == [ { targets = [ "auth.t.local" ]; } ]; } { # Same for the metrics store: one job, by its published name. name = "a host running the metrics store and the collector scrapes the store once, by name"; ok = let o = allLocal.services.hyperhive.swarm.otel; jobs = lib.filter (j: j.job_name == "victoriametrics") (otelSettings allLocal).receivers.prometheus.config.scrape_configs; in allLocal.services.hyperhive.deploy.victoriametrics.enable && allLocal.services.hyperhive.deploy.swarm-otel.enable && !(o.scrapeTargets ? victoriametrics) && (o.publishedScrapeTargets.victoriametrics or null) == vmMetricsUrl && lib.length jobs == 1 && (lib.head jobs).static_configs == [ { targets = [ "metrics.t.local" ]; } ]; } { # The case above reads the options; this one reads the assertion they # feed, which is what refuses an operator's `nixos-rebuild`. name = "a single-host swarm passes the loopback-vs-published job name assertion"; ok = !(refusedOver allLocal "is declared as both a loopback"); } { name = "a collector on a host without the log store scrapes it by name"; ok = let s = otelSettings otelRemoteAuthelia; job = lib.findFirst ( j: j.job_name == "victorialogs" ) null s.receivers.prometheus.config.scrape_configs; in !otelRemoteAuthelia.services.hyperhive.deploy.victorialogs.enable && job != null && job.static_configs == [ { targets = [ "logs.t.local" ]; } ] && job.metrics_path == "/metrics" && job.oauth2.endpoint_params.audience == logsMetricsUrl; } { # `= /metrics` must carry the bare authelia check and nothing else: the # agents' `/__swarm_logs_auth` would admit an agent's bearer, and the # browser block's `error_page 401 =302` hands a scraper a login page. # The other routes keep their own checks. name = "the log store's vhost serves /metrics behind authelia alone, its other routes unchanged"; ok = let h = allLocal.services.hyperhive; locs = allLocal.services.nginx.virtualHosts.${h.swarm.victorialogs.domain}.locations; checksOf = l: lib.filter (lib.hasPrefix "auth_request") (lib.splitString "\n" l.extraConfig); in h.deploy.victorialogs.enable && locs ? "= /metrics" && locs."= /metrics".proxyPass == "http://127.0.0.1:${toString h.swarm.victorialogs.port}/metrics" && checksOf locs."= /metrics" == [ "auth_request /__hive_authelia;" ] && !(lib.hasInfix "error_page" locs."= /metrics".extraConfig) && lib.hasInfix "error_page 401 =302" locs."/".extraConfig && lib.elem "auth_request /__hive_authelia;" (checksOf locs."/") && checksOf locs."^~ /select/logsql/" == [ "auth_request /__swarm_logs_auth;" ]; } { # The `deny` that confines the log store's `/metrics` is scoped to that # path, and comes after the collector's rule: every other path on the # name still reaches `default_policy`, as it did with no rule at all. name = "authelia confines the log store's /metrics to the collector and leaves its other paths alone"; ok = let h = autheliaOnly.services.hyperhive; collector = "oauth2:client:${h.swarm.otel.clientId}"; rules = autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules; logsRules = lib.filter (r: r.domain == "logs.t.local") rules; policy = subject: autheliaPolicy autheliaOnly subject "logs.t.local"; in map (r: r.policy) logsRules == [ "one_factor" "deny" ] && lib.all (r: r.resources == [ "^/metrics([?].*)?$" ]) logsRules && (lib.head logsRules).subject == [ collector ] && policy collector "/metrics" == "one_factor" && policy "user:someone" "/metrics" == "deny" && policy "user:someone" "/metrics?x=1" == "deny" && policy "user:someone" "/" == "one_factor" && policy "user:someone" "/select/logsql/query" == "one_factor" && policy "user:someone" "/insert/opentelemetry/v1/logs" == "one_factor"; } { # Single host: the store is scraped once, by its published name. name = "a host running the log store and the collector scrapes it once, by name"; ok = let o = allLocal.services.hyperhive.swarm.otel; jobs = lib.filter (j: j.job_name == "victorialogs") (otelSettings allLocal).receivers.prometheus.config.scrape_configs; in allLocal.services.hyperhive.deploy.victorialogs.enable && allLocal.services.hyperhive.deploy.swarm-otel.enable && !(o.scrapeTargets ? victorialogs) && lib.length jobs == 1 && (lib.head jobs).static_configs == [ { targets = [ "logs.t.local" ]; } ]; } { name = "a collector on a host without grafana scrapes it by name"; ok = let s = otelSettings otelRemoteAuthelia; job = lib.findFirst (j: j.job_name == "grafana") null s.receivers.prometheus.config.scrape_configs; in !otelRemoteAuthelia.services.hyperhive.deploy.grafana.enable && job != null && job.static_configs == [ { targets = [ "grafana.t.local" ]; } ] && job.metrics_path == "/metrics" && job.oauth2.endpoint_params.audience == grafanaMetricsUrl; } { # The `deny` that confines grafana's `/metrics` is scoped to that path, # and comes after the collector's rule: every other path on the name # still reaches `default_policy`, as it did with no rule at all. name = "authelia confines grafana's /metrics to the collector and leaves its other paths alone"; ok = let h = autheliaOnly.services.hyperhive; collector = "oauth2:client:${h.swarm.otel.clientId}"; rules = autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules; grafanaRules = lib.filter (r: r.domain == "grafana.t.local") rules; policy = subject: autheliaPolicy autheliaOnly subject "grafana.t.local"; in map (r: r.policy) grafanaRules == [ "one_factor" "deny" ] && lib.all (r: r.resources == [ "^/metrics([?].*)?$" ]) grafanaRules && (lib.head grafanaRules).subject == [ collector ] && policy collector "/metrics" == "one_factor" && policy "user:someone" "/metrics" == "deny" && policy "user:someone" "/metrics?x=1" == "deny" && policy "group:admins" "/metrics" == "deny" && policy "user:someone" "/" == "one_factor" && policy "user:someone" "/login/generic_oauth" == "one_factor"; } { # Single host: grafana is scraped once, by its published name. name = "a host running grafana and the collector scrapes it once, by name"; ok = let o = allLocal.services.hyperhive.swarm.otel; jobs = lib.filter (j: j.job_name == "grafana") (otelSettings allLocal).receivers.prometheus.config.scrape_configs; in allLocal.services.hyperhive.deploy.grafana.enable && allLocal.services.hyperhive.deploy.swarm-otel.enable && !(o.scrapeTargets ? grafana) && (o.publishedScrapeTargets.grafana or null) == grafanaMetricsUrl && lib.length jobs == 1 && (lib.head jobs).static_configs == [ { targets = [ "grafana.t.local" ]; } ]; } { name = "a collector on a host without the queue scrapes it by name"; ok = let s = otelSettings otelRemoteAuthelia; job = lib.findFirst (j: j.job_name == "nats") null s.receivers.prometheus.config.scrape_configs; in !otelRemoteAuthelia.services.hyperhive.deploy.nats.enable && job != null && job.static_configs == [ { targets = [ "nats.t.local" ]; } ] && job.metrics_path == "/metrics" && job.oauth2.endpoint_params.audience == natsMetricsUrl; } { # The `deny` that confines the queue's `/metrics` is scoped to that path, # and comes after the collector's rule. name = "authelia confines the queue's /metrics to the collector"; ok = let h = autheliaOnly.services.hyperhive; collector = "oauth2:client:${h.swarm.otel.clientId}"; rules = autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules; natsRules = lib.filter (r: r.domain == "nats.t.local") rules; policy = subject: autheliaPolicy autheliaOnly subject "nats.t.local"; in map (r: r.policy) natsRules == [ "one_factor" "deny" ] && lib.all (r: r.resources == [ "^/metrics([?].*)?$" ]) natsRules && (lib.head natsRules).subject == [ collector ] && policy collector "/metrics" == "one_factor" && policy "user:someone" "/metrics" == "deny" && policy "user:someone" "/metrics?x=1" == "deny" && policy "group:admins" "/metrics" == "deny"; } { # Single host: the queue is scraped once, by its published name. name = "a host running the queue and the collector scrapes it once, by name"; ok = let o = allLocal.services.hyperhive.swarm.otel; jobs = lib.filter (j: j.job_name == "nats") (otelSettings allLocal).receivers.prometheus.config.scrape_configs; in allLocal.services.hyperhive.deploy.nats.enable && allLocal.services.hyperhive.deploy.swarm-otel.enable && !(o.scrapeTargets ? nats) && (o.publishedScrapeTargets.nats or null) == natsMetricsUrl && lib.length jobs == 1 && (lib.head jobs).static_configs == [ { targets = [ "nats.t.local" ]; } ]; } { # Read against the roster the fixture declares rather than against # names spelled here: an arm naming `h1` passes on a single-hive # config however the mapping is written. The length clause is what # makes the `all` mean anything — over an empty roster it holds # vacuously. name = "the swarm collector routes every hive's logs, not just one"; ok = let p = (otelSettings otelTwoHives).service.pipelines; hives = lib.attrNames otelTwoHives.services.hyperhive.swarm.hives; in lib.length hives == 2 && lib.all (h: (p ? "logs/${h}") && p."logs/${h}".receivers == [ "otlp/${h}" ]) hives; } { # The same split as the metrics case above — defining an exporter and # naming it are two lists — plus the half one shared list cannot have: # the metrics store's exporter renders perfectly well inside a logs # pipeline and posts journal records at an ingest route that is not # for them. name = "every logs pipeline sends to the log store and to no metrics one"; ok = let s = otelSettings otelTwoHives; logPipes = lib.filterAttrs (n: _: lib.hasPrefix "logs/" n) s.service.pipelines; used = lib.unique (lib.concatMap (p: p.exporters) (lib.attrValues logPipes)); in logPipes != { } && builtins.elem "otlphttp/victorialogs" used && !(builtins.elem "otlphttp/victoriametrics" used) && lib.all (e: s.exporters ? ${e}) used; } { # 🩸 The refusal, and the only way this module's omission is visible at # all: the reading unit is gated on its own leaf, so the regression is a # green build with the unit absent — and the missing unit is the evidence. # Read as a refusal naming both options, so an operator acts on the # message without opening the nix. name = "a store reader missing the collector's own leaf is refused, naming both options"; ok = refusedOver otelReaderMissingLeaf "otelOidcClientCertFile" && refusedOver otelReaderMissingLeaf "otelOidcClientKeyFile"; } { # ⚠️ The arm that keeps the refusal from eating the degrade beside it. A # collector on a host with NO store identity still receives every hive's # telemetry and is a supported deployment — `otelNoIdentity` is that # fixture, and it differs from the refused one only in the hive's own # leaf. Widening the gate to the collector's leaf alone would reject it, # which is what this pins. name = "a collector with no store identity at all is not refused over the collector's leaf"; ok = !(refusedOver otelNoIdentity "otelOidcClientCertFile") && !(refusedOver otelNoIdentity "otelOidcClientKeyFile"); } { # The other two shapes that must stay buildable: the same remote reader # named in full, and a host that mints its own PKI and therefore has all # eight as defaults from ./host-modules/glue-bao-tls.nix. name = "neither a fully-named remote reader nor a store host is refused"; ok = !(refusedOver otelReaderNamedInFull "otelOidcClientCertFile") && !(refusedOver otelRemoteAuthelia "otelOidcClientCertFile"); } { # The store's units exist only on its host, so elsewhere the fetch waits # on nothing local. The arm where the store is here is in # ./swarm-services-switch.nix. name = "the remote reader's fetch names none of the store's units"; ok = let u = otelReaderNamedInFull.systemd.services.swarm-bao-otel-oidc; storeUnits = [ "swarm-bao-pki.service" "container@${otelReaderNamedInFull.services.hyperhive.swarm.bao.machine}.service" ]; in !(lib.any (n: lib.elem n (u.after ++ u.wants ++ u.requires)) storeUnits); } { # An account in no group, on the swarm UI and the store's UI. A group # rule alone does not refuse it: it skips the rule and lands on # `default_policy`. `swarmctl --swarm`'s bearer is an operator's own # token, so it is the `group:admins` arm here. The forge has no rule and # stays open to any session. name = "authelia refuses an account outside the operator group on the swarm UI and the store's UI"; ok = let h = allLocal.services.hyperhive; ui = h.swarm.bao.ui.domain; policy = autheliaPolicy allLocal; paths = [ "/" "/api/agents" "/api/whoami" "/api/docs/" ]; in h.deploy.swarm-controller.enable && h.swarm.domain == "t.local" && ui == "bao-ui.t.local" && lib.all (p: policy "user:nobody" "t.local" p == "deny") paths && policy "user:nobody" ui "/" == "deny" && lib.all (p: policy "group:admins" "t.local" p == "one_factor") paths && policy "group:admins" ui "/" == "one_factor" && policy "user:nobody" "forge.t.local" "/" == "one_factor"; } { # Authelia on a host that runs no swarm controller and no hive, so the # swarm UI's vhost and the dashboard's are elsewhere. Both still ask # this authelia, so their pairs render here regardless. name = "authelia confines the swarm UI and each hive's dashboard to operators on a host running neither"; ok = let h = autheliaOnly.services.hyperhive; rules = autheliaRules autheliaOnly; policy = autheliaPolicy autheliaOnly; in h.deploy.authelia.enable && !h.deploy.swarm-controller.enable && !h.deploy.hive-controller.enable && lib.filter (r: r.domain == "t.local") rules == [ { domain = "t.local"; subject = [ "group:admins" ]; policy = "one_factor"; } { domain = "t.local"; policy = "deny"; } ] && autheliaSubjectRulesClosed rules && policy "user:nobody" "t.local" "/api/agents" == "deny" && policy "group:admins" "t.local" "/api/agents" == "one_factor" && policy "user:nobody" "h1.hives.t.local" "/api/" == "deny" && policy "group:admins" "h1.hives.t.local" "/api/" == "one_factor"; } { # Every subject-scoped rule the host renders, read in order. The `elem` # clauses are the control for the predicate's vacuous case; the # synthetic lists pin what it accepts and refuses, the first of them # being the real rules with the operator denies cut out. name = "every subject-scoped authelia rule is closed by a deny on its domain"; ok = let rules = autheliaRules allLocal; scoped = lib.filter (r: r ? subject) rules; closed = autheliaSubjectRulesClosed; rejects = rs: !(closed rs); allow = { domain = "t.local"; subject = [ "group:admins" ]; policy = "one_factor"; }; deny = { domain = "t.local"; policy = "deny"; }; narrow = deny // { resources = [ "^/m$" ]; }; operatorDeny = r: r.policy == "deny" && !(r ? subject) && !(r ? resources) && lib.elem r.domain [ "t.local" "bao-ui.t.local" ]; in closed rules && lib.elem allow scoped && lib.elem (allow // { domain = "bao-ui.t.local"; }) scoped && lib.any ( r: r.subject == [ "oauth2:client:${allLocal.services.hyperhive.swarm.otel.clientId}" ] ) scoped && rejects (lib.filter (r: !(operatorDeny r)) rules) && closed [ allow deny ] # a rule for anyone needs no deny behind it && closed [ (removeAttrs allow [ "subject" ]) ] && closed [ (allow // { resources = narrow.resources; }) narrow ] # a narrower deny between two allows refuses everyone it matches && closed [ (allow // { subject = [ "oauth2:client:c" ]; }) narrow allow deny ] && rejects [ allow ] && rejects [ allow narrow ] && rejects [ allow (deny // { domain = "other.local"; }) ] && rejects [ allow (deny // { subject = [ "user:x" ]; }) ] # a subject-less allow in between admits everyone first && rejects [ allow (deny // { policy = "one_factor"; }) deny ] # nothing reaches a rule after a domain-wide deny && rejects [ deny allow ] && rejects [ allow deny (allow // { subject = [ "group:other" ]; }) deny ]; } ]; in runGroup "swarm-otel-identity" cases