diff --git a/docs/networking/gateway.md b/docs/networking/gateway.md index a84c9ecb..768df671 100644 --- a/docs/networking/gateway.md +++ b/docs/networking/gateway.md @@ -19,7 +19,6 @@ You rarely switch it on yourself. `gateway.enable` defaults to off, and every mo | `chat./_matrix/*` | tuwunel (`8008`) | `hive-matrix.nix`, `swarm.matrix.gatewayHost != null` | | `chat./` | fluffychat-web static | `hive-matrix.nix`, `swarm.matrix.gatewayHost != null` | | `chat./config.json` | inline JSON (FluffyChat boot config) | `hive-matrix.nix`, `swarm.matrix.gatewayHost != null` | -| `/metrics` | the queue exporter's `/metrics` on loopback, behind an authelia `auth_request`; every other path, `/` included, returns `403`, so `/varz` and `/connz` are never reachable | `swarm-nats.nix`, the queue's host | | `grafana.`, `metrics.`, `logs.`, `otel.`, `bao.` | the matching swarm service | that service's module → [`swarm/services.md`](../swarm/services.md) | **The hive's own vhost**, named for the hive domain. It exists only on a host that runs a hive (`deploy.hive-controller.enable`); a host without a hive has none of these rows: diff --git a/docs/swarm/README.md b/docs/swarm/README.md index be229c74..eb61c462 100644 --- a/docs/swarm/README.md +++ b/docs/swarm/README.md @@ -132,14 +132,6 @@ reader `deploy.bao.clientCertFile`: issue a leaf whose subject is do, `swarm-bao-nats-tls` fails naming both whenever the queue's certificate is missing or due for renewal. -The queue host's gateway also serves the queue's metrics, at -`https:///metrics`, so a swarm collector on any host -scrapes them. Only the collector's authelia client gets through; any other -session gets `deny`, and every other path on that name is 403. With the -gateway's default self-signed TLS, that vhost serves the queue's own -certificate, so a host whose only swarm service is the queue needs no -swarm-services certificate. - The identity isn't a choice — a hive authenticates as `hive-` and publishes under `hiveName`, the same name that keys `swarm.hives`. diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index 48bbba74..44e0d638 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -43,29 +43,28 @@ neither is a renaming of the other. ## Swarm-level — one of each per swarm -| secret | generated by | lives at | hive elsewhere | -| -------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| swarm-services root CA (cert + key) | `swarm-bao.nix`'s granting unit, inside the store's `pki` mount | the **key** never leaves the mount; the **cert** lands at `/var/lib/swarm-bao-services-pki/services-root.pem` (0644) and at `/var/lib/hive-tls/swarm-services-root.pem` on every hive | nothing to copy — every hive receives the cert with the leaf it asks the store for, or from `pki/ca/pem` where it fronts no swarm name and needs a store identity | -| authelia session, JWT and storage-encryption keys | authelia's first-boot unit, in-container | `/var/lib/authelia-swarm/{session,jwt,storage-encryption}.key` | generated in place; nothing outside that container reads them | -| authelia OIDC HMAC key | same unit | `/var/lib/authelia-swarm/oidc-hmac.key` | same | -| authelia OIDC issuer key (RSA) | same unit | `/var/lib/authelia-swarm/oidc-issuer.key` | same — relying parties verify against the **public** half at `/jwks.json` | -| OIDC client secret, plaintext half | `authelia crypto hash generate --random` | `/var/lib/authelia-swarm/oidc-clients/.secret` | operator provides the file and names it in whichever option reads it — `sso.clientSecretFile` for a service; the forge and, by default, the hive collector need none — `swarm-bao-forge-oidc` and `swarm-bao-hive-otel-oidc` fetch it from the store (below) | -| OIDC client secret, digest half | the same mint | `oidc-clients/.digest` | authelia's own half; merged at runtime via `settingsFiles` | -| the swarm collector's copy of its OIDC secret | `swarm-bao-otel-oidc.service` reads it out of the swarm secret store, **on every host that runs the collector and holds a store identity** | `/var/lib/swarm-otel-oidc/.secret` inside the `swarm-otel` container | same unit, same path — one route, co-located or not. A collector with no store identity has `clientSecretFile == null`, its already-supported unauthenticated-push degrade — see below | -| Grafana's copy of its OIDC secret | `swarm-bao-grafana-oidc.service` reads it out of the swarm secret store, **on every host that runs Grafana** | `/var/lib/grafana-oidc/.secret` inside the `swarm-grafana` container | same unit, same path — one route, co-located or not. Nothing for an operator to place beyond this host's store leaf, see below | -| Grafana's admin password | `swarm-bao-grafana-admin.service` creates it in the swarm secret store at `swarm/services//admin/password` when the store holds none, then copies it out, **on every host that runs Grafana**. Grafana's leaf may create that leaf but not update it, so the store keeps the first value, and the secret publisher can't write it. To rotate, run `bao kv metadata delete secret/swarm/services//admin/password` with a token that may, such as the root token (Grafana's, the publisher's and the controller's can't), then `systemctl restart swarm-bao-grafana-admin`; a plain `bao kv delete` isn't enough, see [the callout xkey's section](#the-queues-callout-xkey) | `/var/lib/grafana-admin/password` inside the `swarm-grafana` container: `security.admin_password` for a new database, and `grafana cli admin reset-admin-password` in `grafana.service`'s `preStart` for an existing one, on every start | same unit, same path. Grafana doesn't start until the file exists, so it stays down rather than falling back to the upstream default `admin` | -| the swarm controller's copy of its OIDC secret | `swarm-controller` reads it out of the swarm secret store at start, under its own leaf | the daemon's memory only; never on disk | same read, co-located or not. Nothing for an operator to place beyond the controller's store leaf (`deploy.swarm-controller.baoClientCertFile`) | -| a hive's copy of its own OIDC secret | hive-c0re reads it out of the swarm secret store when it first connects to the queue, under the hive's own leaf | the daemon's memory only; never on disk | same read, co-located or not. Nothing for an operator to place beyond the hive's store leaf (`deploy.bao.clientCertFile`) | -| the hive collector's copy of its hive's OIDC secret | `swarm-bao-hive-otel-oidc.service` reads it out of the swarm secret store under the hive's own leaf, **on every hive that runs the collector and holds that leaf** | `/var/lib/hive-otel-oidc/hive-.secret` on the hive's host, 0400 root, loaded by `LoadCredential` | same unit, same path, co-located or not. Nothing for an operator to place beyond the hive's store leaf (`deploy.bao.clientCertFile`) | -| the swarm controller's forge token | the forge's `forgejo-swarm-controller-account`, in-container, once | `/var/lib/forgejo/swarm-controller-token` inside the forge container, the minter's record that it has minted; `swarm-bao-forge-publisher` copies it into the swarm secret store at `swarm/controller/swarm-controller/forge/token` each time the forge container starts and each time the mint writes the file; the controller holds it in memory only | nothing to place beyond the controller's store leaf. A forge host without the store needs the publisher's leaf placed (`deploy.bao.forgePublisherClientCertFile`) | -| the queue responder's copy of its OIDC secret | the responder reads it out of the swarm secret store itself as it starts, under its own leaf, and again when authelia refuses it | the daemon's memory only; never on disk | same read, wherever authelia runs. Nothing for an operator to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | -| authelia subject store | `swarmctl` and `swarm-authelia-bridge` | `users.yml` — one file, read and written by both | `swarmctl`, on the host that runs authelia | -| wireguard private key | **the operator** — `wg genkey` | whatever `deploy.wireguard.privateKeyFile` names | always operator-provided; nothing generates this for you | -| queue auth-callout nkeys (user seed + account seed) | `swarm-nats-auth ensure-callout-keys`, run by `swarm-bao-nats-callout-xkey` on the queue's host when the store holds no seed; the verb stores a seed it finds at `/var/lib/swarm-nats-callout/{callout-user,issuer}.seed` as is, then removes the file | the swarm secret store: the seeds at `swarm/services/swarm-nats/callout/{user,issuer}-seed`, read by the `swarm-nats-auth` role alone; the public halves at `swarm/public/swarm-nats/callout-{user,issuer}` | nothing to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | -| queue auth-callout xkey (curve seed) | `swarm-nats-auth ensure-callout-keys`, run by `swarm-bao-nats-callout-xkey` on the queue's host when the store holds no seed | the swarm secret store: the seed at `swarm/services/swarm-nats/callout/xkey-seed`, read by the `swarm-nats-auth` role alone; the public half at `swarm/public/swarm-nats/callout-xkey` | nothing to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | -| the queue's TLS key, copied for the gateway's `/metrics` vhost | `swarm-nats-gateway-cert.service` on the queue's host, where the gateway uses self-signed TLS: a copy of the queue's bao-PKI TLS leaf (`/var/lib/swarm-nats-tls/`) once `swarm-bao-nats-tls` has issued it, else a throwaway 30-day self-signed pair for the queue's name until then | `/var/lib/swarm-nats-gateway-tls/{cert,key}.pem`; the key is `0640` group `nginx`, read by nginx for the queue's `/metrics` vhost. The queue's own key stays `0600` root | the copy is redone on each leaf renewal (`swarm-bao-nats-tls` starts the unit, which reloads nginx if it's active); a restart before any leaf exists keeps the pair already there. Until `swarm-bao-nats-tls` issues the leaf the vhost serves the throwaway cert, so the collector's scrape fails TLS. Nothing for an operator to place | -| the secret store's own contents | openbao, on first `bao operator init` — **an operator action, not a unit** | inside the `swarm-bao` container, at its own `/var/lib/openbao`, kept across rebuilds by `ephemeral = false`. ⚠️ **Not a host path**: `nixos-container destroy swarm-bao` takes the raft data with it, so back up the container's tree, not `/var/lib/`. Only the store's TLS material (`/var/lib/swarm-bao-tls`) and its PKCS11 token (`/var/lib/swarm-bao-token`) are host-level | n/a — there is one store; a hive elsewhere is a _client_ of it and holds none of this | -| the secret store's unseal material | the HSM/TPM under `deploy.bao.seal = "pkcs11"`; openbao itself under `"shamir"` | in the token; or held by whoever ran `bao operator init`, which is what `"shamir"` means and why it's stated rather than inferred | n/a — only the host running the store seals anything | +| secret | generated by | lives at | hive elsewhere | +| --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| swarm-services root CA (cert + key) | `swarm-bao.nix`'s granting unit, inside the store's `pki` mount | the **key** never leaves the mount; the **cert** lands at `/var/lib/swarm-bao-services-pki/services-root.pem` (0644) and at `/var/lib/hive-tls/swarm-services-root.pem` on every hive | nothing to copy — every hive receives the cert with the leaf it asks the store for, or from `pki/ca/pem` where it fronts no swarm name and needs a store identity | +| authelia session, JWT and storage-encryption keys | authelia's first-boot unit, in-container | `/var/lib/authelia-swarm/{session,jwt,storage-encryption}.key` | generated in place; nothing outside that container reads them | +| authelia OIDC HMAC key | same unit | `/var/lib/authelia-swarm/oidc-hmac.key` | same | +| authelia OIDC issuer key (RSA) | same unit | `/var/lib/authelia-swarm/oidc-issuer.key` | same — relying parties verify against the **public** half at `/jwks.json` | +| OIDC client secret, plaintext half | `authelia crypto hash generate --random` | `/var/lib/authelia-swarm/oidc-clients/.secret` | operator provides the file and names it in whichever option reads it — `sso.clientSecretFile` for a service; the forge and, by default, the hive collector need none — `swarm-bao-forge-oidc` and `swarm-bao-hive-otel-oidc` fetch it from the store (below) | +| OIDC client secret, digest half | the same mint | `oidc-clients/.digest` | authelia's own half; merged at runtime via `settingsFiles` | +| the swarm collector's copy of its OIDC secret | `swarm-bao-otel-oidc.service` reads it out of the swarm secret store, **on every host that runs the collector and holds a store identity** | `/var/lib/swarm-otel-oidc/.secret` inside the `swarm-otel` container | same unit, same path — one route, co-located or not. A collector with no store identity has `clientSecretFile == null`, its already-supported unauthenticated-push degrade — see below | +| Grafana's copy of its OIDC secret | `swarm-bao-grafana-oidc.service` reads it out of the swarm secret store, **on every host that runs Grafana** | `/var/lib/grafana-oidc/.secret` inside the `swarm-grafana` container | same unit, same path — one route, co-located or not. Nothing for an operator to place beyond this host's store leaf, see below | +| Grafana's admin password | `swarm-bao-grafana-admin.service` creates it in the swarm secret store at `swarm/services//admin/password` when the store holds none, then copies it out, **on every host that runs Grafana**. Grafana's leaf may create that leaf but not update it, so the store keeps the first value, and the secret publisher can't write it. To rotate, run `bao kv metadata delete secret/swarm/services//admin/password` with a token that may, such as the root token (Grafana's, the publisher's and the controller's can't), then `systemctl restart swarm-bao-grafana-admin`; a plain `bao kv delete` isn't enough, see [the callout xkey's section](#the-queues-callout-xkey) | `/var/lib/grafana-admin/password` inside the `swarm-grafana` container: `security.admin_password` for a new database, and `grafana cli admin reset-admin-password` in `grafana.service`'s `preStart` for an existing one, on every start | same unit, same path. Grafana doesn't start until the file exists, so it stays down rather than falling back to the upstream default `admin` | +| the swarm controller's copy of its OIDC secret | `swarm-controller` reads it out of the swarm secret store at start, under its own leaf | the daemon's memory only; never on disk | same read, co-located or not. Nothing for an operator to place beyond the controller's store leaf (`deploy.swarm-controller.baoClientCertFile`) | +| a hive's copy of its own OIDC secret | hive-c0re reads it out of the swarm secret store when it first connects to the queue, under the hive's own leaf | the daemon's memory only; never on disk | same read, co-located or not. Nothing for an operator to place beyond the hive's store leaf (`deploy.bao.clientCertFile`) | +| the hive collector's copy of its hive's OIDC secret | `swarm-bao-hive-otel-oidc.service` reads it out of the swarm secret store under the hive's own leaf, **on every hive that runs the collector and holds that leaf** | `/var/lib/hive-otel-oidc/hive-.secret` on the hive's host, 0400 root, loaded by `LoadCredential` | same unit, same path, co-located or not. Nothing for an operator to place beyond the hive's store leaf (`deploy.bao.clientCertFile`) | +| the swarm controller's forge token | the forge's `forgejo-swarm-controller-account`, in-container, once | `/var/lib/forgejo/swarm-controller-token` inside the forge container, the minter's record that it has minted; `swarm-bao-forge-publisher` copies it into the swarm secret store at `swarm/controller/swarm-controller/forge/token` each time the forge container starts and each time the mint writes the file; the controller holds it in memory only | nothing to place beyond the controller's store leaf. A forge host without the store needs the publisher's leaf placed (`deploy.bao.forgePublisherClientCertFile`) | +| the queue responder's copy of its OIDC secret | the responder reads it out of the swarm secret store itself as it starts, under its own leaf, and again when authelia refuses it | the daemon's memory only; never on disk | same read, wherever authelia runs. Nothing for an operator to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | +| authelia subject store | `swarmctl` and `swarm-authelia-bridge` | `users.yml` — one file, read and written by both | `swarmctl`, on the host that runs authelia | +| wireguard private key | **the operator** — `wg genkey` | whatever `deploy.wireguard.privateKeyFile` names | always operator-provided; nothing generates this for you | +| queue auth-callout nkeys (user seed + account seed) | `swarm-nats-auth ensure-callout-keys`, run by `swarm-bao-nats-callout-xkey` on the queue's host when the store holds no seed; the verb stores a seed it finds at `/var/lib/swarm-nats-callout/{callout-user,issuer}.seed` as is, then removes the file | the swarm secret store: the seeds at `swarm/services/swarm-nats/callout/{user,issuer}-seed`, read by the `swarm-nats-auth` role alone; the public halves at `swarm/public/swarm-nats/callout-{user,issuer}` | nothing to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | +| queue auth-callout xkey (curve seed) | `swarm-nats-auth ensure-callout-keys`, run by `swarm-bao-nats-callout-xkey` on the queue's host when the store holds no seed | the swarm secret store: the seed at `swarm/services/swarm-nats/callout/xkey-seed`, read by the `swarm-nats-auth` role alone; the public half at `swarm/public/swarm-nats/callout-xkey` | nothing to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | +| the secret store's own contents | openbao, on first `bao operator init` — **an operator action, not a unit** | inside the `swarm-bao` container, at its own `/var/lib/openbao`, kept across rebuilds by `ephemeral = false`. ⚠️ **Not a host path**: `nixos-container destroy swarm-bao` takes the raft data with it, so back up the container's tree, not `/var/lib/`. Only the store's TLS material (`/var/lib/swarm-bao-tls`) and its PKCS11 token (`/var/lib/swarm-bao-token`) are host-level | n/a — there is one store; a hive elsewhere is a _client_ of it and holds none of this | +| the secret store's unseal material | the HSM/TPM under `deploy.bao.seal = "pkcs11"`; openbao itself under `"shamir"` | in the token; or held by whoever ran `bao operator init`, which is what `"shamir"` means and why it's stated rather than inferred | n/a — only the host running the store seals anything | Authelia mints the three keys for itself, in-container, precisely because nothing outside that container ever reads them. **That's the test worth diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index ba169276..cf88a4f5 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -44,6 +44,7 @@ ./glue-nats-bao-identity.nix ./glue-nats-oidc-client.nix ./glue-nats-wireguard.nix + ./glue-otel-scrape-targets.nix ./glue-queue-agent-credential.nix ./glue-builder-oidc-client.nix ./glue-review-classifier-oidc-client.nix diff --git a/nix/host-modules/glue-otel-scrape-targets.nix b/nix/host-modules/glue-otel-scrape-targets.nix new file mode 100644 index 00000000..e01e8204 --- /dev/null +++ b/nix/host-modules/glue-otel-scrape-targets.nix @@ -0,0 +1,37 @@ +# Glue: declare the loopback scrape target of nats on the swarm's OTEL +# collector, wherever it shares a host with it. Authelia and grafana are +# `publishedScrapeTargets` jobs (./swarm-otel-service.nix); a loopback job of +# the same name fails ./swarm-otel.nix's job-name assertion. +# +# ONE PAIRING — nats ← the collector. Deleting this leaves the collector not +# scraping nats: no error, no metrics. +# +# The target is gated on the service AND the collector being enabled here. The +# loopback address is only honest when both run on this host; a target nobody +# can read asserts a collection that is not happening. Both default from +# `deploy.allSwarmServices`, but `mkDefault` is an invitation to override, so +# co-location is a property of the auto-deployed topology, not the modules. +# +# ⚠️ Silent when split: separate hosts are separate evaluations with no shared +# context, so a host running nats and no collector declares nothing and +# the collector's host never learns of it. No error, no warning — a +# healthy exporter and an empty dashboard. Not guardable from here. +# +# ⚠️ `swarm.otel`, not `hyperhive.otel` — two collectors one word apart, and only +# this one reads `scrapeTargets`. +{ + lib, + config, + ... +}: +let + hyperhiveCfg = config.services.hyperhive; + deployCfg = hyperhiveCfg.deploy; + swarmCfg = hyperhiveCfg.swarm; + withCollector = service: lib.mkIf (deployCfg.${service}.enable && deployCfg.swarm-otel.enable); +in +{ + config.services.hyperhive.swarm.otel.scrapeTargets = { + nats = withCollector "nats" "127.0.0.1:${toString swarmCfg.nats.metricsPort}"; + }; +} diff --git a/nix/host-modules/swarm-nats-service.nix b/nix/host-modules/swarm-nats-service.nix index aeeea857..56ec72cf 100644 --- a/nix/host-modules/swarm-nats-service.nix +++ b/nix/host-modules/swarm-nats-service.nix @@ -40,11 +40,8 @@ in Every other hive resolves it through the operator's upstream DNS, which is where a multi-host swarm needs a record for it. - Not in {option}`services.hyperhive.swarm.serviceDomains`. The gateway - on the queue's host serves `https:///metrics` to the swarm - collector, and in self-signed mode it serves the queue's own leaf - there rather than the swarm-services one, so that host never waits - for a services leaf (./swarm-nats.nix). + Not in {option}`services.hyperhive.swarm.serviceDomains`: that list + is the names a gateway vhost fronts, and nginx fronts nothing here. ''; }; @@ -92,9 +89,7 @@ in default = 7777; description = '' Port the Prometheus exporter serves NATS's metrics on, bound to - loopback. The gateway's `= /metrics` location on - {option}`services.hyperhive.swarm.nats.domain` proxies it, behind - the swarm collector's authelia check. + loopback for the swarm collector to scrape. ⚠️ This and {option}`monitorPort` are two more claims on a port space every swarm container shares — they run in this container but diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index 2f285e52..5bc0d64a 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -286,14 +286,6 @@ let tlsKeyCredential = "tls-key"; tlsKeyCredentialPath = "/run/credentials/nats.service/${tlsKeyCredential}"; - # The gateway's copy of that leaf, for the `/metrics` vhost below: the key - # here is 0640 root:nginx, so nginx reads it while the queue's own stays - # 0600 root. Self-signed mode only, the mode `gateway.lib.tlsFor` hands the - # swarm-services leaf to; with ACME or an operator cert that issuer covers - # this name as it covers every other. - gatewayServesQueueLeaf = gatewayCfg.enable && gatewayCfg.useSelfSigned; - gatewayTlsDir = "/var/lib/swarm-nats-gateway-tls"; - # The responder's own store identity, for the callout xkey, agent queue # credentials and its OIDC client secret. Required: an assertion below refuses # a queue without it. @@ -337,12 +329,12 @@ in # The swarm's message queue: one NATS server, reached by every hive at # `tls://:`. # - # ⚠️ The gateway does NOT front the queue. NATS speaks its own TCP protocol - # rather than HTTP, so the server terminates TLS itself on `port`. The + # ⚠️ There is deliberately NO gateway vhost here. NATS speaks its own TCP + # protocol rather than HTTP, so nginx cannot front it the way it fronts the + # forge, matrix and authelia, and the server terminates TLS itself. The # name is the half of the sibling pattern that applies, the way it is for # bao: `gateway.localNames` on this host, the operator's DNS everywhere - # else. The one gateway vhost on that name serves the exporter's - # `/metrics` to the swarm collector and nothing else. + # else. # What the queue IS to every hive, the ports it answers on and the client id # it is registered under, is `swarm.nats` in ./swarm-nats-service.nix. What @@ -437,124 +429,14 @@ in }; config = lib.mkIf deployCfg.nats.enable { - # The auth responder introspects authelia by name, and the gateway serves - # the exporter's `/metrics` below. - services.hyperhive.gateway.enable = lib.mkDefault true; + # Resolver only: NATS speaks its own protocol, so nginx fronts + # nothing here — but the auth responder introspects authelia by name. services.hyperhive.gateway.dns.enable = lib.mkDefault true; # The name every client dials, answered with the bridge IP on this host. # Other hives resolve it through the operator's DNS, as they do bao's. services.hyperhive.gateway.localNames = [ cfg.domain ]; - # The exporter's metrics, for the swarm collector wherever it runs, and - # nothing else on this name: the monitoring endpoint (`/varz`, `/connz`) - # is never proxied, and every other path is 403. - # - # ⚠️ Not `gateway.lib.tlsFor`, and the name is not in - # `swarm.serviceDomains`. In self-signed mode this vhost serves the - # queue's own leaf (issued for exactly this name, `server_flag` set), so - # a host whose only swarm service is the queue never fronts a services - # name and its gateway does not wait for the swarm-services leaf. - # - # `forceSSL` for the reason ./swarm-victorialogs.nix gives: authelia - # answers a plain-http auth subrequest with 400. - services.nginx.virtualHosts."${cfg.domain}" = - ( - if gatewayServesQueueLeaf then - { - sslCertificate = "${gatewayTlsDir}/cert.pem"; - sslCertificateKey = "${gatewayTlsDir}/key.pem"; - } - else - builtins.removeAttrs (gatewayCfg.lib.tlsFor cfg.domain) [ "addSSL" ] - ) - // { - forceSSL = true; - listen = gatewayCfg.lib.listen; - extraConfig = gatewayCfg.lib.securityHeaders; - locations = { - "/".return = "403"; - - # The collector requests exactly this URL, declared as - # `publishedScrapeTargets.nats` in ./swarm-otel-service.nix; the - # two must agree, since that string is also the token's audience. - # Authelia's rules for it admit the collector's client and deny - # every other subject (`metricsRules` in ./swarm-authelia.nix). - "= /metrics" = { - proxyPass = "http://127.0.0.1:${toString cfg.metricsPort}/metrics"; - extraConfig = '' - auth_request /__nats_metrics_authz; - ''; - }; - - # The subrequest, same shape as the forge's `= /__forge_metrics_authz` - # (./hive-forge/default.nix): no `error_page 401 =302`, so a scraper - # gets the 401 rather than a login page parsed as metrics. - "= /__nats_metrics_authz" = { - proxyPass = "https://${autheliaCfg.domain}/api/authz/auth-request"; - # nixpkgs appends its OWN `Host $host` after extraConfig, which - # would override verifiedProxyTo's — see hive-gateway/vhost-lib.nix. - recommendedProxySettings = false; - extraConfig = '' - internal; - ${gatewayCfg.lib.verifiedProxyTo autheliaCfg.domain} - proxy_pass_request_body off; - proxy_set_header Content-Length ""; - proxy_set_header X-Original-Method $request_method; - proxy_set_header X-Original-URL $scheme://$http_host$request_uri; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Host $http_host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - ''; - }; - }; - }; - - # Puts a pair at the paths the vhost above names, before nginx starts: - # nginx refuses a config naming a missing cert file, and that takes the - # whole gateway down rather than this one name. The queue's leaf when it - # exists, else a throwaway self-signed pair until `swarm-bao-nats-tls` - # issues the leaf and starts this again. Never fails on a missing leaf. - systemd.services.swarm-nats-gateway-cert = lib.mkIf gatewayServesQueueLeaf { - description = "Copy the swarm queue's TLS leaf for the gateway's /metrics vhost"; - wantedBy = [ "nginx.service" ]; - before = [ "nginx.service" ]; - path = [ - pkgs.coreutils - pkgs.openssl - pkgs.systemd - ]; - serviceConfig = { - Type = "oneshot"; - UMask = "0077"; - SyslogIdentifier = "swarm-nats-gateway-cert"; - }; - script = '' - set -eu - src=${lib.escapeShellArg tlsDir} - d=${lib.escapeShellArg gatewayTlsDir} - install -d -m 0755 "$d" - if [ -s "$src/cert.pem" ] && [ -s "$src/key.pem" ]; then - cp "$src/cert.pem" "$d/cert.pem.new" - cp "$src/key.pem" "$d/key.pem.new" - elif [ -s "$d/cert.pem" ] && [ -s "$d/key.pem" ]; then - exit 0 - else - echo "no queue leaf yet — serving a throwaway self-signed certificate on ${cfg.domain} until swarm-bao-nats-tls issues one" >&2 - openssl req -x509 -newkey rsa:2048 -nodes -sha256 -days 30 \ - -subj "/CN=${cfg.domain}" -keyout "$d/key.pem.new" -out "$d/cert.pem.new" - fi - chmod 0644 "$d/cert.pem.new" - chgrp nginx "$d/key.pem.new" - chmod 0640 "$d/key.pem.new" - mv -f "$d/key.pem.new" "$d/key.pem" - mv -f "$d/cert.pem.new" "$d/cert.pem" - if systemctl is-active --quiet nginx.service; then - systemctl reload nginx.service - fi - ''; - }; - assertions = [ { # The callout xkey and the responder's OIDC client secret are read @@ -1117,10 +999,6 @@ in chmod 0644 "$d/cert.pem.new" mv -f "$d/key.pem.new" "$d/key.pem" mv -f "$d/cert.pem.new" "$d/cert.pem" - ${lib.optionalString gatewayServesQueueLeaf '' - - # The gateway's `/metrics` vhost serves a copy of this leaf. - systemctl start swarm-nats-gateway-cert.service''} # Renewal, or the late-store retry. On a normal boot the container is # not up yet and starts the server with this leaf itself. A restart, diff --git a/nix/host-modules/swarm-otel-service.nix b/nix/host-modules/swarm-otel-service.nix index 9b7edcf0..63b992fb 100644 --- a/nix/host-modules/swarm-otel-service.nix +++ b/nix/host-modules/swarm-otel-service.nix @@ -240,8 +240,8 @@ in declared on a host that does not serve it. An entry that also needs the collector on the same host (the loopback address is only honest then) is declared from a glue module instead, which reads both - switches. Every swarm service is a `publishedScrapeTargets` entry - instead, which a collector on any host scrapes. + switches: grafana and nats are declared by + `glue-otel-scrape-targets.nix`. ⚠️ **That constrains the TARGET, not the SCRAPER, and the difference is a silent gap.** Nothing here puts the collector on the same host. @@ -364,7 +364,5 @@ in victorialogs = "https://${hyperhiveCfg.swarm.victorialogs.domain}/metrics"; # The vhost location is in ./swarm-grafana.nix. grafana = "https://${hyperhiveCfg.swarm.grafana.domain}/metrics"; - # The vhost location is in ./swarm-nats.nix. - nats = "https://${hyperhiveCfg.swarm.nats.domain}/metrics"; }; } diff --git a/nix/host-modules/swarm.nix b/nix/host-modules/swarm.nix index ee393eba..e91b780c 100644 --- a/nix/host-modules/swarm.nix +++ b/nix/host-modules/swarm.nix @@ -93,10 +93,6 @@ let # services leaf over the hive one, so a gateway-published swarm name # missing here is served the hive certificate, which cannot cover a name # under a different apex. - # - # `swarm.nats.domain` is deliberately absent: its `/metrics` vhost names - # the queue's own leaf instead (./swarm-nats.nix), so a host running only - # the queue fronts no services name and never waits for that leaf. serviceDomains' = [ swarmCfg.forge.domain swarmCfg.matrix.gatewayHost diff --git a/nix/module-eval/forge-placement.nix b/nix/module-eval/forge-placement.nix index 49579f65..f04b5dcd 100644 --- a/nix/module-eval/forge-placement.nix +++ b/nix/module-eval/forge-placement.nix @@ -138,16 +138,15 @@ let && !(refusedOver bare "deploy.forgejo.sso.clientSecretFile"); } { - # All three are published targets; ./swarm-otel-identity.nix covers them. - name = "a host running the collector, authelia, grafana and nats scrapes none of them on loopback"; + # Authelia and grafana are published targets; ./swarm-otel-identity.nix + # covers them. + name = "a host running the collector, authelia, grafana and nats scrapes nats on loopback, authelia and grafana not"; ok = let t = servicesHere.services.hyperhive.swarm.otel.scrapeTargets; + port = n: toString servicesHere.services.hyperhive.swarm.${n}.metricsPort; in - servicesHere.services.hyperhive.deploy.nats.enable - && !(t ? authelia) - && !(t ? grafana) - && !(t ? nats); + t.nats == "127.0.0.1:${port "nats"}" && !(t ? authelia) && !(t ? grafana); } { name = "hosting the swarm's shared services runs the forge"; diff --git a/nix/module-eval/nats-tls.nix b/nix/module-eval/nats-tls.nix index b64d3695..c0301447 100644 --- a/nix/module-eval/nats-tls.nix +++ b/nix/module-eval/nats-tls.nix @@ -362,74 +362,6 @@ let failedAssertions remoteLeafNoUrl == [ ] && !(remoteLeafNoUrl.systemd.services.hive-c0re.environment ? HIVE_C0RE_NATS_URL); } - { - # Only `/metrics`, from the exporter, behind the collector's subrequest: - # no `error_page` (a scraper would parse a login page as metrics), the - # monitoring endpoint never proxied, every other path refused. In - # self-signed mode the vhost names the queue's own leaf, as copied for - # nginx, not the swarm-services one. - name = "the queue's name serves only /metrics, behind the collector's auth subrequest, on the queue's own leaf"; - ok = - let - h = allLocal.services.hyperhive; - v = allLocal.services.nginx.virtualHosts.${natsName}; - locs = v.locations; - checksOf = l: lib.filter (lib.hasPrefix "auth_request") (lib.splitString "\n" l.extraConfig); - authz = locs."= /__nats_metrics_authz"; - in - lib.attrNames locs == [ - "/" - "= /__nats_metrics_authz" - "= /metrics" - ] - && locs."/".return == "403" - && locs."/".proxyPass == null - && locs."= /metrics".proxyPass == "http://127.0.0.1:${toString h.swarm.nats.metricsPort}/metrics" - && checksOf locs."= /metrics" == [ "auth_request /__nats_metrics_authz;" ] - && !(lib.hasInfix "error_page" locs."= /metrics".extraConfig) - && authz.proxyPass == "https://auth.t.local/api/authz/auth-request" - && lib.hasInfix "internal;" authz.extraConfig - && !(lib.any ( - l: - lib.hasInfix ":${toString h.swarm.nats.monitorPort}" ( - if l.proxyPass == null then "" else l.proxyPass - ) - ) (lib.attrValues locs)) - && v.forceSSL - && v.sslCertificate == "/var/lib/swarm-nats-gateway-tls/cert.pem" - && v.sslCertificateKey == "/var/lib/swarm-nats-gateway-tls/key.pem"; - } - { - # A host whose only swarm service is the queue: its gateway serves the - # queue's name with the queue's leaf, fronts no swarm-services name, and - # so neither waits for nor requires the swarm-services leaf. The copy - # unit is only wanted by nginx, never required, and a rotated queue leaf - # re-runs it. - name = "a host running only the queue renders its gateway without requiring the swarm-services leaf"; - ok = - let - h = queueNoStore.services.hyperhive; - s = queueNoStore.systemd.services; - copy = s.swarm-nats-gateway-cert; - in - h.deploy.nats.enable - && !h.deploy.hive-controller.enable - && h.gateway.enable - && h.gateway.useSelfSigned - && - queueNoStore.services.nginx.virtualHosts.${natsName}.sslCertificate - == "/var/lib/swarm-nats-gateway-tls/cert.pem" - && h.swarm.localServiceDomains == [ ] - && !(lib.elem natsName h.swarm.serviceDomains) - && !(lib.elem "hive-gateway-self-signed-cert.service" (s.swarm-services-cert.requiredBy or [ ])) - && !(lib.hasInfix "serves swarm names only with it" s.hive-gateway-self-signed-cert.script) - && lib.elem "nginx.service" copy.wantedBy - && lib.elem "nginx.service" copy.before - && !(lib.elem "nginx.service" (copy.requiredBy or [ ])) - && lib.hasInfix "throwaway self-signed" copy.script - && lib.hasInfix "chgrp nginx" copy.script - && lib.hasInfix "systemctl start swarm-nats-gateway-cert.service" s.swarm-bao-nats-tls.script; - } ]; in runGroup "nats-tls" cases diff --git a/nix/module-eval/swarm-otel-identity.nix b/nix/module-eval/swarm-otel-identity.nix index 2d27d63c..04c0d047 100644 --- a/nix/module-eval/swarm-otel-identity.nix +++ b/nix/module-eval/swarm-otel-identity.nix @@ -111,7 +111,6 @@ let grafanaMetricsUrl = "https://grafana.t.local/metrics"; logsMetricsUrl = "https://logs.t.local/metrics"; vmMetricsUrl = "https://metrics.t.local/metrics"; - natsMetricsUrl = "https://nats.t.local/metrics"; # The policy authelia applies to `subject` requesting `https://`, # where `path` is what authelia matches `resources` against: the cleaned path @@ -379,9 +378,8 @@ let grafanaMetricsUrl logsMetricsUrl vmMetricsUrl - natsMetricsUrl ] - && lib.length targets == 12 + && lib.length targets == 10 && lib.all (t: policy collector t == "one_factor") targets && lib.all (t: policy "user:someone" t == "deny") targets && lib.all (t: policy "group:admins" t == "deny") targets; @@ -429,7 +427,7 @@ let ]; matched = row: builtins.match re row.path != null; in - lib.length metricsRules == 10 + lib.length metricsRules == 8 && lib.all (r: r.resources == [ re ]) metricsRules && re == "^/metrics([?].*)?$" && lib.all (row: matched row == (row.nginx == "= /metrics")) table; @@ -623,60 +621,6 @@ let && lib.length jobs == 1 && (lib.head jobs).static_configs == [ { targets = [ "grafana.t.local" ]; } ]; } - { - name = "a collector on a host without the queue scrapes it by name"; - ok = - let - s = otelSettings otelRemoteAuthelia; - job = lib.findFirst (j: j.job_name == "nats") null s.receivers.prometheus.config.scrape_configs; - in - !otelRemoteAuthelia.services.hyperhive.deploy.nats.enable - && job != null - && job.static_configs == [ { targets = [ "nats.t.local" ]; } ] - && job.metrics_path == "/metrics" - && job.oauth2.endpoint_params.audience == natsMetricsUrl; - } - { - # The `deny` that confines the queue's `/metrics` is scoped to that path, - # and comes after the collector's rule. - name = "authelia confines the queue's /metrics to the collector"; - ok = - let - h = autheliaOnly.services.hyperhive; - collector = "oauth2:client:${h.swarm.otel.clientId}"; - rules = - autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules; - natsRules = lib.filter (r: r.domain == "nats.t.local") rules; - policy = subject: autheliaPolicy autheliaOnly subject "nats.t.local"; - in - map (r: r.policy) natsRules == [ - "one_factor" - "deny" - ] - && lib.all (r: r.resources == [ "^/metrics([?].*)?$" ]) natsRules - && (lib.head natsRules).subject == [ collector ] - && policy collector "/metrics" == "one_factor" - && policy "user:someone" "/metrics" == "deny" - && policy "user:someone" "/metrics?x=1" == "deny" - && policy "group:admins" "/metrics" == "deny"; - } - { - # Single host: the queue is scraped once, by its published name. - name = "a host running the queue and the collector scrapes it once, by name"; - ok = - let - o = allLocal.services.hyperhive.swarm.otel; - jobs = - lib.filter (j: j.job_name == "nats") - (otelSettings allLocal).receivers.prometheus.config.scrape_configs; - in - allLocal.services.hyperhive.deploy.nats.enable - && allLocal.services.hyperhive.deploy.swarm-otel.enable - && !(o.scrapeTargets ? nats) - && (o.publishedScrapeTargets.nats or null) == natsMetricsUrl - && lib.length jobs == 1 - && (lib.head jobs).static_configs == [ { targets = [ "nats.t.local" ]; } ]; - } { # Read against the roster the fixture declares rather than against # names spelled here: an arm naming `h1` passes on a single-hive