diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index c1b7fea2..232e7459 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -58,6 +58,24 @@ let # swarm-tier pipeline would be added here and inherit the check for free. reservedOwners = [ swarmTierName ]; + # A published target is declared as ONE url, because that url is also the + # audience its token is minted for — but prometheus wants the same fact in + # three fields. Split it here rather than asking a service to state it + # twice: two spellings of one address is a mismatch waiting to happen, and + # the failure is a valid token refused at the target. + # + # `null` when the shape is wrong, which the assertion below reports by name. + # ⚠️ The pattern requires `https`. A published endpoint reached over plain + # http would ship a bearer token in clear text, and that is worth an eval + # error rather than a warning nobody reads. + parsePublished = url: builtins.match "https://([^/]+)(/.*)" url; + + # Where the collector reads its own client secret. Under /var/lib and not + # /run for the same reason the forge and matrix secrets are: the collector + # may start before the delivery unit on a later boot, and a secret that + # evaporates on reboot turns a working scrape into an intermittent one. + collectorSecretPath = "/var/lib/swarm-otel/${cfg.clientId}.secret"; + # `attrNames` is sorted, so this is a function of the hive SET and not of # the order anyone wrote it in. # @@ -421,6 +439,28 @@ in Rename the hive. Reserved: ${lib.concatMapStringsSep ", " (n: "'${n}'") reservedOwners}. ''; } + { + # A published target that is not an `https://host/path` url. Without + # this the split returns null and the failure surfaces as + # `attempt to call elemAt on null` from inside the renderer, naming + # neither the option nor the offending value. + # + # It also enforces the scheme: a published endpoint scraped over plain + # http would put a bearer token on the wire in clear text. + assertion = lib.all (u: parsePublished u != null) (lib.attrValues cfg.publishedScrapeTargets); + message = '' + services.hyperhive.swarm.otel.publishedScrapeTargets has ${ + lib.concatMapStringsSep ", " (kv: "${kv.name} = \"${kv.value}\"") ( + lib.filter (kv: parsePublished kv.value == null) (lib.attrsToList cfg.publishedScrapeTargets) + ) + }, which is not of the form https:///. + + The value is both the address scraped and the audience the + collector's token is minted for, so it has to be the exact url a + request goes to. https is required: this hop carries a bearer + token, and plain http would put it on the wire in clear text. + ''; + } { # A job name used by BOTH scrape options. Within one option this # cannot happen — the module system refuses two definitions of the @@ -600,11 +640,48 @@ in # target: a `prometheus` receiver with nothing to scrape is # the shape this whole issue is about, a config that renders # and deploys perfectly while adding no data. - // lib.optionalAttrs (cfg.scrapeTargets != { }) { - prometheus.config.scrape_configs = lib.mapAttrsToList (job: target: { - job_name = job; - static_configs = [ { targets = [ target ]; } ]; - }) cfg.scrapeTargets; + // lib.optionalAttrs (cfg.scrapeTargets != { } || cfg.publishedScrapeTargets != { }) { + prometheus.config.scrape_configs = + lib.mapAttrsToList (job: target: { + job_name = job; + static_configs = [ { targets = [ target ]; } ]; + }) cfg.scrapeTargets + # ⚠️ `++`, so the two kinds of target land in ONE list — + # which is exactly why a job name may not appear in both + # options. A list concatenation does not resolve a + # collision the way an attrset would: both entries ship + # under the same `job_name`. The assertion above is what + # stands between that and a deploy. + ++ lib.mapAttrsToList ( + job: url: + let + parts = parsePublished url; + in + { + job_name = job; + # Split from the declared url — see `parsePublished`. + scheme = "https"; + static_configs = [ { targets = [ (lib.elemAt parts 0) ]; } ]; + metrics_path = lib.elemAt parts 1; + # Prometheus-native oauth2, not the collector's + # `oauth2client` extension: the prometheus receiver + # takes upstream's scrape config verbatim and does not + # accept an `auth` block naming a collector extension. + oauth2 = { + client_id = cfg.clientId; + # A path, never a value — nothing here may read the + # secret, or it lands in the store world-readable. + client_secret_file = collectorSecretPath; + token_url = "${autheliaCfg.url}/api/oidc/token"; + # The audience is the target's own url, and authelia + # checks it against the address being requested. + # Registered ≠ requested: a client that does not ASK + # for an audience gets a token with `aud: []` however + # complete its registration looks. + endpoint_params.audience = url; + }; + } + ) cfg.publishedScrapeTargets; }; exporters =