diff --git a/Cargo.lock b/Cargo.lock index 4a496e0b..592d785a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4943,6 +4943,22 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "swarm-metrics-auth" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "clap", + "serde", + "subtle", + "swarm-queue-client", + "swarm-secret-client", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "swarm-nats-auth" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 00087010..096d4bd8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ members = [ "swarm-review-classifier", "swarm-logs", "swarm-logs-auth", + "swarm-metrics-auth", "swarm-secret-client", "swarmctl", ] diff --git a/docs/process/gotchas.md b/docs/process/gotchas.md index 55a1c546..e2ee26f2 100644 --- a/docs/process/gotchas.md +++ b/docs/process/gotchas.md @@ -82,7 +82,8 @@ With both set, systemd creates the runtime directory under `/run/private/` `RuntimeDirectoryPreserve=`). No unprivileged user can traverse that, so nginx's `connect()` to a socket in it fails with `EACCES` whatever the directory and socket modes are. A unit serving a socket to another user -runs as a static user instead (`swarm-controller`, `swarm-logs-auth`). +runs as a static user instead (`swarm-controller`, `swarm-logs-auth`, +`swarm-metrics-auth`). `module-eval-core-toggle` fails on any `unix:` upstream of nginx owned by such a unit. diff --git a/docs/swarm/credentials.md b/docs/swarm/credentials.md index ba520a0f..7fd613a3 100644 --- a/docs/swarm/credentials.md +++ b/docs/swarm/credentials.md @@ -74,6 +74,7 @@ of the cell says how. | `swarm/hives//oidc/client` | authelia, at its first boot; `swarm-secret-publish` copies it in | hive-c0re, under the hive's own certificate, when it first connects to the queue | ❌ authelia mints it once. A re-mint is republished by `swarm-secret-publish`'s path unit | ✅ read until the first connect succeeds, retried with a backoff of up to five minutes while it's missing; then held in memory and read again when authelia refuses it | | `swarm/services//oidc/client` | authelia | that service's own reader, under its own certificate — the queue responder's under `swarm-nats-auth`'s, the homeserver's under `swarm-matrix-ctl`'s, the forge's under `swarm-forge-publisher`'s, the review classifier's under `swarm-review-classifier`'s; the bao UI's by the store's granter | must be stated | must be stated | | `swarm/services/ci-runner/registration-token` | `swarm-controller`, from the forge's admin API, at start and in a five-minute pass; its grant on the path is write-only | `swarm-bao-ci-runner` on the runner's host, under the runner's own certificate, into a file only `swarm-ci-runner-register` in the runner's container loads; no other principal's policy reads it | ❌ the forge hands out one token until an admin resets it; the controller writes a reset token within five minutes | ✅ `swarm-bao-ci-runner` reads it at boot, on deploy and every ten minutes, and restarts the runner when the value changed | +| `swarm/services//metrics/bearer` | `swarm-bao-grafana-metrics-bearer` on Grafana's host, under Grafana's certificate, when the store holds none | `swarm-metrics-auth` on the metrics store's host, under its own certificate, at most once a minute, to check the bearer Grafana's datasource presents at the store's read API | ❌ nothing re-mints it; an operator deletes it and restarts `swarm-bao-grafana-metrics-bearer` | ✅ `swarm-metrics-auth` reads it again once its copy is a minute old. ❌ Grafana reads its file copy only as it starts; the bearer unit restarts Grafana when the copy changes | | `swarm/services/review-classifier/anthropic-api-key` | the operator, by hand in the store's UI, field `api_key` | `swarm-review-classifier` in its container, under its own certificate, at the start of every API key run; held for that run and handed to `claude`'s environment only; no other principal's policy reads it | ❌ an operator's key; replaced by writing a new one | ✅ read on every run, so a written or removed key takes effect on the next run | | `swarm/services/review-classifier/acp-provider` | the operator, by hand in the store's UI, field `api_key` | `swarm-review-classifier` in its container, under its own certificate, at the start of every ACP run; held for that run and handed to `opencode`'s environment only; no other principal's policy reads it | ❌ an operator's key; replaced by writing a new one | ✅ read on every run, so a written or removed key takes effect on the next run | | `swarm/services/review-classifier/forge/token` | `swarm-controller`, with the forge's admin API, for its `review-classifier` account, at start and in a five-minute pass; the secret publisher's policy denies it | `swarm-controller` under its own certificate, before it decides whether to mint, and on every review it posts as the account; the classifier service's policy doesn't read it | ✅ the controller's pass re-mints when the stored token is missing or no longer matches the forge (last eight characters and scopes) | ✅ the controller reads the store on every review it posts | diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index 48bbba74..0c590ac2 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -43,29 +43,31 @@ neither is a renaming of the other. ## Swarm-level — one of each per swarm -| secret | generated by | lives at | hive elsewhere | -| -------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| swarm-services root CA (cert + key) | `swarm-bao.nix`'s granting unit, inside the store's `pki` mount | the **key** never leaves the mount; the **cert** lands at `/var/lib/swarm-bao-services-pki/services-root.pem` (0644) and at `/var/lib/hive-tls/swarm-services-root.pem` on every hive | nothing to copy — every hive receives the cert with the leaf it asks the store for, or from `pki/ca/pem` where it fronts no swarm name and needs a store identity | -| authelia session, JWT and storage-encryption keys | authelia's first-boot unit, in-container | `/var/lib/authelia-swarm/{session,jwt,storage-encryption}.key` | generated in place; nothing outside that container reads them | -| authelia OIDC HMAC key | same unit | `/var/lib/authelia-swarm/oidc-hmac.key` | same | -| authelia OIDC issuer key (RSA) | same unit | `/var/lib/authelia-swarm/oidc-issuer.key` | same — relying parties verify against the **public** half at `/jwks.json` | -| OIDC client secret, plaintext half | `authelia crypto hash generate --random` | `/var/lib/authelia-swarm/oidc-clients/.secret` | operator provides the file and names it in whichever option reads it — `sso.clientSecretFile` for a service; the forge and, by default, the hive collector need none — `swarm-bao-forge-oidc` and `swarm-bao-hive-otel-oidc` fetch it from the store (below) | -| OIDC client secret, digest half | the same mint | `oidc-clients/.digest` | authelia's own half; merged at runtime via `settingsFiles` | -| the swarm collector's copy of its OIDC secret | `swarm-bao-otel-oidc.service` reads it out of the swarm secret store, **on every host that runs the collector and holds a store identity** | `/var/lib/swarm-otel-oidc/.secret` inside the `swarm-otel` container | same unit, same path — one route, co-located or not. A collector with no store identity has `clientSecretFile == null`, its already-supported unauthenticated-push degrade — see below | -| Grafana's copy of its OIDC secret | `swarm-bao-grafana-oidc.service` reads it out of the swarm secret store, **on every host that runs Grafana** | `/var/lib/grafana-oidc/.secret` inside the `swarm-grafana` container | same unit, same path — one route, co-located or not. Nothing for an operator to place beyond this host's store leaf, see below | -| Grafana's admin password | `swarm-bao-grafana-admin.service` creates it in the swarm secret store at `swarm/services//admin/password` when the store holds none, then copies it out, **on every host that runs Grafana**. Grafana's leaf may create that leaf but not update it, so the store keeps the first value, and the secret publisher can't write it. To rotate, run `bao kv metadata delete secret/swarm/services//admin/password` with a token that may, such as the root token (Grafana's, the publisher's and the controller's can't), then `systemctl restart swarm-bao-grafana-admin`; a plain `bao kv delete` isn't enough, see [the callout xkey's section](#the-queues-callout-xkey) | `/var/lib/grafana-admin/password` inside the `swarm-grafana` container: `security.admin_password` for a new database, and `grafana cli admin reset-admin-password` in `grafana.service`'s `preStart` for an existing one, on every start | same unit, same path. Grafana doesn't start until the file exists, so it stays down rather than falling back to the upstream default `admin` | -| the swarm controller's copy of its OIDC secret | `swarm-controller` reads it out of the swarm secret store at start, under its own leaf | the daemon's memory only; never on disk | same read, co-located or not. Nothing for an operator to place beyond the controller's store leaf (`deploy.swarm-controller.baoClientCertFile`) | -| a hive's copy of its own OIDC secret | hive-c0re reads it out of the swarm secret store when it first connects to the queue, under the hive's own leaf | the daemon's memory only; never on disk | same read, co-located or not. Nothing for an operator to place beyond the hive's store leaf (`deploy.bao.clientCertFile`) | -| the hive collector's copy of its hive's OIDC secret | `swarm-bao-hive-otel-oidc.service` reads it out of the swarm secret store under the hive's own leaf, **on every hive that runs the collector and holds that leaf** | `/var/lib/hive-otel-oidc/hive-.secret` on the hive's host, 0400 root, loaded by `LoadCredential` | same unit, same path, co-located or not. Nothing for an operator to place beyond the hive's store leaf (`deploy.bao.clientCertFile`) | -| the swarm controller's forge token | the forge's `forgejo-swarm-controller-account`, in-container, once | `/var/lib/forgejo/swarm-controller-token` inside the forge container, the minter's record that it has minted; `swarm-bao-forge-publisher` copies it into the swarm secret store at `swarm/controller/swarm-controller/forge/token` each time the forge container starts and each time the mint writes the file; the controller holds it in memory only | nothing to place beyond the controller's store leaf. A forge host without the store needs the publisher's leaf placed (`deploy.bao.forgePublisherClientCertFile`) | -| the queue responder's copy of its OIDC secret | the responder reads it out of the swarm secret store itself as it starts, under its own leaf, and again when authelia refuses it | the daemon's memory only; never on disk | same read, wherever authelia runs. Nothing for an operator to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | -| authelia subject store | `swarmctl` and `swarm-authelia-bridge` | `users.yml` — one file, read and written by both | `swarmctl`, on the host that runs authelia | -| wireguard private key | **the operator** — `wg genkey` | whatever `deploy.wireguard.privateKeyFile` names | always operator-provided; nothing generates this for you | -| queue auth-callout nkeys (user seed + account seed) | `swarm-nats-auth ensure-callout-keys`, run by `swarm-bao-nats-callout-xkey` on the queue's host when the store holds no seed; the verb stores a seed it finds at `/var/lib/swarm-nats-callout/{callout-user,issuer}.seed` as is, then removes the file | the swarm secret store: the seeds at `swarm/services/swarm-nats/callout/{user,issuer}-seed`, read by the `swarm-nats-auth` role alone; the public halves at `swarm/public/swarm-nats/callout-{user,issuer}` | nothing to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | -| queue auth-callout xkey (curve seed) | `swarm-nats-auth ensure-callout-keys`, run by `swarm-bao-nats-callout-xkey` on the queue's host when the store holds no seed | the swarm secret store: the seed at `swarm/services/swarm-nats/callout/xkey-seed`, read by the `swarm-nats-auth` role alone; the public half at `swarm/public/swarm-nats/callout-xkey` | nothing to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | -| the queue's TLS key, copied for the gateway's `/metrics` vhost | `swarm-nats-gateway-cert.service` on the queue's host, where the gateway uses self-signed TLS: a copy of the queue's bao-PKI TLS leaf (`/var/lib/swarm-nats-tls/`) once `swarm-bao-nats-tls` has issued it, else a throwaway 30-day self-signed pair for the queue's name until then | `/var/lib/swarm-nats-gateway-tls/{cert,key}.pem`; the key is `0640` group `nginx`, read by nginx for the queue's `/metrics` vhost. The queue's own key stays `0600` root | the copy is redone on each leaf renewal (`swarm-bao-nats-tls` starts the unit, which reloads nginx if it's active); a restart before any leaf exists keeps the pair already there. Until `swarm-bao-nats-tls` issues the leaf the vhost serves the throwaway cert, so the collector's scrape fails TLS. Nothing for an operator to place | -| the secret store's own contents | openbao, on first `bao operator init` — **an operator action, not a unit** | inside the `swarm-bao` container, at its own `/var/lib/openbao`, kept across rebuilds by `ephemeral = false`. ⚠️ **Not a host path**: `nixos-container destroy swarm-bao` takes the raft data with it, so back up the container's tree, not `/var/lib/`. Only the store's TLS material (`/var/lib/swarm-bao-tls`) and its PKCS11 token (`/var/lib/swarm-bao-token`) are host-level | n/a — there is one store; a hive elsewhere is a _client_ of it and holds none of this | -| the secret store's unseal material | the HSM/TPM under `deploy.bao.seal = "pkcs11"`; openbao itself under `"shamir"` | in the token; or held by whoever ran `bao operator init`, which is what `"shamir"` means and why it's stated rather than inferred | n/a — only the host running the store seals anything | +| secret | generated by | lives at | hive elsewhere | +| -------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| swarm-services root CA (cert + key) | `swarm-bao.nix`'s granting unit, inside the store's `pki` mount | the **key** never leaves the mount; the **cert** lands at `/var/lib/swarm-bao-services-pki/services-root.pem` (0644) and at `/var/lib/hive-tls/swarm-services-root.pem` on every hive | nothing to copy — every hive receives the cert with the leaf it asks the store for, or from `pki/ca/pem` where it fronts no swarm name and needs a store identity | +| authelia session, JWT and storage-encryption keys | authelia's first-boot unit, in-container | `/var/lib/authelia-swarm/{session,jwt,storage-encryption}.key` | generated in place; nothing outside that container reads them | +| authelia OIDC HMAC key | same unit | `/var/lib/authelia-swarm/oidc-hmac.key` | same | +| authelia OIDC issuer key (RSA) | same unit | `/var/lib/authelia-swarm/oidc-issuer.key` | same — relying parties verify against the **public** half at `/jwks.json` | +| OIDC client secret, plaintext half | `authelia crypto hash generate --random` | `/var/lib/authelia-swarm/oidc-clients/.secret` | operator provides the file and names it in whichever option reads it — `sso.clientSecretFile` for a service; the forge and, by default, the hive collector need none — `swarm-bao-forge-oidc` and `swarm-bao-hive-otel-oidc` fetch it from the store (below) | +| OIDC client secret, digest half | the same mint | `oidc-clients/.digest` | authelia's own half; merged at runtime via `settingsFiles` | +| the swarm collector's copy of its OIDC secret | `swarm-bao-otel-oidc.service` reads it out of the swarm secret store, **on every host that runs the collector and holds a store identity** | `/var/lib/swarm-otel-oidc/.secret` inside the `swarm-otel` container | same unit, same path — one route, co-located or not. A collector with no store identity has `clientSecretFile == null`, its already-supported unauthenticated-push degrade — see below | +| Grafana's copy of its OIDC secret | `swarm-bao-grafana-oidc.service` reads it out of the swarm secret store, **on every host that runs Grafana** | `/var/lib/grafana-oidc/.secret` inside the `swarm-grafana` container | same unit, same path — one route, co-located or not. Nothing for an operator to place beyond this host's store leaf, see below | +| Grafana's admin password | `swarm-bao-grafana-admin.service` creates it in the swarm secret store at `swarm/services//admin/password` when the store holds none, then copies it out, **on every host that runs Grafana**. Grafana's leaf may create that leaf but not update it, so the store keeps the first value, and the secret publisher can't write it. To rotate, run `bao kv metadata delete secret/swarm/services//admin/password` with a token that may, such as the root token (Grafana's, the publisher's and the controller's can't), then `systemctl restart swarm-bao-grafana-admin`; a plain `bao kv delete` isn't enough, see [the callout xkey's section](#the-queues-callout-xkey) | `/var/lib/grafana-admin/password` inside the `swarm-grafana` container: `security.admin_password` for a new database, and `grafana cli admin reset-admin-password` in `grafana.service`'s `preStart` for an existing one, on every start | same unit, same path. Grafana doesn't start until the file exists, so it stays down rather than falling back to the upstream default `admin` | +| Grafana's metrics bearer | `swarm-bao-grafana-metrics-bearer.service` creates it in the swarm secret store at `swarm/services//metrics/bearer` when the store holds none, then copies it out, **on every host that runs Grafana**. Same grant as the admin password, create without update. To rotate, run `bao kv metadata delete secret/swarm/services//metrics/bearer` with a token that may, then `systemctl restart swarm-bao-grafana-metrics-bearer` | `/var/lib/grafana-metrics/bearer` inside the `swarm-grafana` container, holding `Bearer `: the metrics datasource's `Authorization` header. `swarm-metrics-auth` on the metrics store's host reads the stored value itself, under its own leaf | same unit, same path. The unit restarts Grafana when the file changes | +| the metrics store responder's own store leaf (cert + key) | `signLeaf` in `glue-bao-tls.nix` (`swarm-bao-pki.service`), on the host that runs the secret store, once, subject `deploy.bao.metricsAuthCommonName`. `swarm-bao-metrics-auth-policy.service` writes the `swarm-metrics-auth` role and its read grant on the bearer leaf, nothing else | `/var/lib/swarm-bao-pki/metrics-auth.pem` and `metrics-auth-key.pem` on the store's host, handed to `swarm-metrics-auth` through `LoadCredential` (paths in argv, never values) | **the operator** places a leaf with that subject on the metrics store's host and names it in `deploy.victoriametrics.authBaoClientCertFile` / `authBaoClientKeyFile`; evaluation fails naming both until then. If the store is on another host, deploy that host first: the role must exist before the responder's login succeeds, and until it does the gateway refuses every Grafana query (fail closed) | +| the swarm controller's copy of its OIDC secret | `swarm-controller` reads it out of the swarm secret store at start, under its own leaf | the daemon's memory only; never on disk | same read, co-located or not. Nothing for an operator to place beyond the controller's store leaf (`deploy.swarm-controller.baoClientCertFile`) | +| a hive's copy of its own OIDC secret | hive-c0re reads it out of the swarm secret store when it first connects to the queue, under the hive's own leaf | the daemon's memory only; never on disk | same read, co-located or not. Nothing for an operator to place beyond the hive's store leaf (`deploy.bao.clientCertFile`) | +| the hive collector's copy of its hive's OIDC secret | `swarm-bao-hive-otel-oidc.service` reads it out of the swarm secret store under the hive's own leaf, **on every hive that runs the collector and holds that leaf** | `/var/lib/hive-otel-oidc/hive-.secret` on the hive's host, 0400 root, loaded by `LoadCredential` | same unit, same path, co-located or not. Nothing for an operator to place beyond the hive's store leaf (`deploy.bao.clientCertFile`) | +| the swarm controller's forge token | the forge's `forgejo-swarm-controller-account`, in-container, once | `/var/lib/forgejo/swarm-controller-token` inside the forge container, the minter's record that it has minted; `swarm-bao-forge-publisher` copies it into the swarm secret store at `swarm/controller/swarm-controller/forge/token` each time the forge container starts and each time the mint writes the file; the controller holds it in memory only | nothing to place beyond the controller's store leaf. A forge host without the store needs the publisher's leaf placed (`deploy.bao.forgePublisherClientCertFile`) | +| the queue responder's copy of its OIDC secret | the responder reads it out of the swarm secret store itself as it starts, under its own leaf, and again when authelia refuses it | the daemon's memory only; never on disk | same read, wherever authelia runs. Nothing for an operator to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | +| authelia subject store | `swarmctl` and `swarm-authelia-bridge` | `users.yml` — one file, read and written by both | `swarmctl`, on the host that runs authelia | +| wireguard private key | **the operator** — `wg genkey` | whatever `deploy.wireguard.privateKeyFile` names | always operator-provided; nothing generates this for you | +| queue auth-callout nkeys (user seed + account seed) | `swarm-nats-auth ensure-callout-keys`, run by `swarm-bao-nats-callout-xkey` on the queue's host when the store holds no seed; the verb stores a seed it finds at `/var/lib/swarm-nats-callout/{callout-user,issuer}.seed` as is, then removes the file | the swarm secret store: the seeds at `swarm/services/swarm-nats/callout/{user,issuer}-seed`, read by the `swarm-nats-auth` role alone; the public halves at `swarm/public/swarm-nats/callout-{user,issuer}` | nothing to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | +| queue auth-callout xkey (curve seed) | `swarm-nats-auth ensure-callout-keys`, run by `swarm-bao-nats-callout-xkey` on the queue's host when the store holds no seed | the swarm secret store: the seed at `swarm/services/swarm-nats/callout/xkey-seed`, read by the `swarm-nats-auth` role alone; the public half at `swarm/public/swarm-nats/callout-xkey` | nothing to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | +| the queue's TLS key, copied for the gateway's `/metrics` vhost | `swarm-nats-gateway-cert.service` on the queue's host, where the gateway uses self-signed TLS: a copy of the queue's bao-PKI TLS leaf (`/var/lib/swarm-nats-tls/`) once `swarm-bao-nats-tls` has issued it, else a throwaway 30-day self-signed pair for the queue's name until then | `/var/lib/swarm-nats-gateway-tls/{cert,key}.pem`; the key is `0640` group `nginx`, read by nginx for the queue's `/metrics` vhost. The queue's own key stays `0600` root | the copy is redone on each leaf renewal (`swarm-bao-nats-tls` starts the unit, which reloads nginx if it's active); a restart before any leaf exists keeps the pair already there. Until `swarm-bao-nats-tls` issues the leaf the vhost serves the throwaway cert, so the collector's scrape fails TLS. Nothing for an operator to place | +| the secret store's own contents | openbao, on first `bao operator init` — **an operator action, not a unit** | inside the `swarm-bao` container, at its own `/var/lib/openbao`, kept across rebuilds by `ephemeral = false`. ⚠️ **Not a host path**: `nixos-container destroy swarm-bao` takes the raft data with it, so back up the container's tree, not `/var/lib/`. Only the store's TLS material (`/var/lib/swarm-bao-tls`) and its PKCS11 token (`/var/lib/swarm-bao-token`) are host-level | n/a — there is one store; a hive elsewhere is a _client_ of it and holds none of this | +| the secret store's unseal material | the HSM/TPM under `deploy.bao.seal = "pkcs11"`; openbao itself under `"shamir"` | in the token; or held by whoever ran `bao operator init`, which is what `"shamir"` means and why it's stated rather than inferred | n/a — only the host running the store seals anything | Authelia mints the three keys for itself, in-container, precisely because nothing outside that container ever reads them. **That's the test worth diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 54a4bc2b..6757ec9e 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -171,14 +171,29 @@ Grafana start — see [`secrets.md`](secrets.md). Neither container is reachable except through the gateway: each listens on a unix socket only the gateway's nginx can open, and VictoriaMetrics runs in a network namespace of its own with no interface but loopback. -VictoriaMetrics authenticates nothing itself. The -`metrics.` vhost forwards only the store's read API (the -Prometheus query, series, label and metadata routes, plus the vmui web UI) -without a credential. It forwards the collector's OTLP push, and the -collector's scrape of the store's own `/metrics`, only for the swarm -collector's authelia client: authelia refuses any other session or client -there. It refuses every other path, the store's write and admin -API included, with 403. The bare domain redirects to vmui. +VictoriaMetrics authenticates nothing itself; the `metrics.` +vhost does it: + +- **The read API** (the Prometheus query, series, label and metadata + routes): Grafana's datasource or an operator. Grafana sends a bearer that + `swarm-bao-grafana-metrics-bearer` creates in the swarm secret store, and + `swarm-metrics-auth` on the store's host checks it against the same store + path. A request without a bearer needs an operator's authelia session + (group `admins`); without one it gets a 401, not a login redirect. +- **vmui**: an operator's authelia session, with a redirect to the login. + The bare domain redirects to vmui. +- **The collector's OTLP push, and its scrape of the store's own + `/metrics`**: only the swarm collector's authelia client. Authelia refuses + any other session or client there, operators included. +- **Every other path**, the store's write and admin API included: 403. + +`swarm-metrics-auth` reads the store with a leaf of its own, +`deploy.victoriametrics.authBaoClientCertFile` / `authBaoClientKeyFile`. +Where the secret store runs on the same host, these default to the leaf it +mints. On any other host you place a leaf whose CN is +`deploy.bao.metricsAuthCommonName`, and evaluation refuses the config until +you set both. It reads the stored bearer at most once a minute, so a +replaced bearer starts working up to a minute later. The collector scrapes Grafana's own metrics at `https:///metrics`, so a collector on a host without diff --git a/flake.nix b/flake.nix index c1e02ea9..b63c8148 100644 --- a/flake.nix +++ b/flake.nix @@ -157,6 +157,9 @@ services.hyperhive.deploy.victorialogs.authPackage = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.swarm-logs-auth; + services.hyperhive.deploy.victoriametrics.authPackage = + lib.mkDefault + self.packages.${pkgs.stdenv.hostPlatform.system}.swarm-metrics-auth; services.hyperhive.deploy.nats.authPackage = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.swarm-nats-auth; diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index ba169276..14cb6129 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -38,6 +38,7 @@ ./glue-hive-otel-authelia.nix ./glue-hive-tls-bao-identity.nix ./glue-logs-auth-bao-identity.nix + ./glue-metrics-auth-bao-identity.nix ./glue-matrix-ctl-bao-identity.nix ./glue-matrix-oidc-client.nix ./glue-nats-auth-bao-identity.nix diff --git a/nix/host-modules/glue-bao-readers-policy-order.nix b/nix/host-modules/glue-bao-readers-policy-order.nix index 2657e961..90412d41 100644 --- a/nix/host-modules/glue-bao-readers-policy-order.nix +++ b/nix/host-modules/glue-bao-readers-policy-order.nix @@ -67,11 +67,15 @@ let ]; }; # ./swarm-grafana.nix, under Grafana's leaf, which may create the admin - # password only once that policy is written. + # password and the metrics bearer only once that policy is written. swarm-bao-grafana-admin = { here = deployCfg.grafana.enable; policies = [ "swarm-bao-grafana-oidc-policy" ]; }; + swarm-bao-grafana-metrics-bearer = { + here = deployCfg.grafana.enable; + policies = [ "swarm-bao-grafana-oidc-policy" ]; + }; # ./hive-matrix.nix, under matrix-ctl's leaf. swarm-bao-matrix-oidc = { here = deployCfg.matrix.enable; diff --git a/nix/host-modules/glue-bao-readers-store-order.nix b/nix/host-modules/glue-bao-readers-store-order.nix index 6a4776e6..0e74d121 100644 --- a/nix/host-modules/glue-bao-readers-store-order.nix +++ b/nix/host-modules/glue-bao-readers-store-order.nix @@ -46,6 +46,7 @@ let # ./swarm-grafana.nix swarm-bao-grafana-oidc = deployCfg.grafana.enable; swarm-bao-grafana-admin = deployCfg.grafana.enable; + swarm-bao-grafana-metrics-bearer = deployCfg.grafana.enable; # ./swarm-nats.nix swarm-bao-nats-callout-xkey = deployCfg.nats.enable; # ./swarm-ci-runner.nix diff --git a/nix/host-modules/glue-bao-tls.nix b/nix/host-modules/glue-bao-tls.nix index e19006d9..482609ce 100644 --- a/nix/host-modules/glue-bao-tls.nix +++ b/nix/host-modules/glue-bao-tls.nix @@ -276,6 +276,13 @@ in [ -s ${pkiDir}/logs-auth.pem ] || ${signLeaf} ${pkiDir} logs-auth \ ${lib.escapeShellArg deployCfg.bao.logsAuthCommonName} "" clientAuth + # The metrics store's Grafana auth responder, which reads Grafana's + # metrics bearer. Minted here for the matrix-ctl leaf's reason: the + # metrics store is a swarm singleton, so elsewhere this is the file an + # operator copies. + [ -s ${pkiDir}/metrics-auth.pem ] || ${signLeaf} ${pkiDir} metrics-auth \ + ${lib.escapeShellArg deployCfg.bao.metricsAuthCommonName} "" clientAuth + # The CI runner's token fetch. Minted here for the matrix-ctl leaf's # reason: the runner is a swarm singleton, so on a runner host without # the store this is the file an operator copies. diff --git a/nix/host-modules/glue-metrics-auth-bao-identity.nix b/nix/host-modules/glue-metrics-auth-bao-identity.nix new file mode 100644 index 00000000..d9efcd75 --- /dev/null +++ b/nix/host-modules/glue-metrics-auth-bao-identity.nix @@ -0,0 +1,32 @@ +# Glue: point the metrics store's Grafana auth responder at the bao leaf minted +# for it. +# +# ONE PAIRING PER FILE — the swarm-metrics-auth principal ← bao, and nothing +# else. Deleting this leaves a metrics store with no responder identity unless +# the operator names one, which ./swarm-victoriametrics.nix refuses at eval. +# +# ⚠️ The minting is NOT here. ./glue-bao-tls.nix holds the CA and signs the +# leaf. What belongs here is the pairing: which paths the responder presents. +# +# ⚠️ Gated on the store running on this host, the only place the leaf is +# minted. A metrics-store host without the store gets no default here; the +# operator places the responder's leaf and names it. +# +# Everything is `mkDefault`. An operator naming their own paths wins. +{ + lib, + config, + ... +}: +let + deployCfg = config.services.hyperhive.deploy; + pkiDir = import ./lib/bao-pki-dir.nix; +in +{ + config = lib.mkIf (deployCfg.victoriametrics.enable && deployCfg.bao.enable) { + services.hyperhive.deploy.victoriametrics = { + authBaoClientCertFile = lib.mkDefault "${pkiDir}/metrics-auth.pem"; + authBaoClientKeyFile = lib.mkDefault "${pkiDir}/metrics-auth-key.pem"; + }; + }; +} diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index 082aa2fb..fc4698ea 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -97,14 +97,43 @@ let ) ++ [ (target // { policy = "deny"; }) ]; - # The metrics store's name, every path. Only its push route and `/metrics` - # consult authelia (its read API is served with no check), and the - # collector is the one caller of both, so the rules take no `resources` - # and leave no path or query form to fall through to `default_policy`. - # ⚠️ An authenticated location added to that vhost for anyone else is - # refused here until it gets a rule ahead of the `deny`. + # The metrics store's name, every path. Authelia is consulted there for the + # collector's push route and `/metrics`, and for an operator's session on + # the read API and vmui; Grafana's bearer on the read API is checked by + # `swarm-metrics-auth`, never here. + # + # The collector's client is admitted anywhere on the name. Every other + # subject is refused the collector's two paths, then operators are admitted + # to the rest, then everything else is refused. Prefixes rather than exact + # paths for the collector's two, so no spelling of them reaches the + # operator rule. No rule takes a narrower `resources` than that, so no path + # or query form falls through to `default_policy`. metricsStoreDomain = config.services.hyperhive.swarm.victoriametrics.domain; - metricsStoreRules = collectorOnly { domain = metricsStoreDomain; }; + metricsStoreRules = + lib.optional collectorRegistered { + domain = metricsStoreDomain; + policy = "one_factor"; + subject = [ "oauth2:client:${collectorClientId}" ]; + } + ++ [ + { + domain = metricsStoreDomain; + policy = "deny"; + resources = [ + "^/opentelemetry/.*" + "^/metrics.*" + ]; + } + { + domain = metricsStoreDomain; + policy = "one_factor"; + subject = [ "group:${operatorGroup}" ]; + } + { + domain = metricsStoreDomain; + policy = "deny"; + } + ]; # One pair per `swarm.otel.publishedScrapeTargets` entry, read off the same # URL the collector requests and its token is minted for, so a target diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 5b6988cd..0cd4145d 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -908,15 +908,17 @@ let # the objects they write. # # Grafana's leaf also creates its admin password (`swarm-bao-grafana-admin` - # in ./swarm-grafana.nix). `create` without `update`, so a stored password is - # never replaced. + # in ./swarm-grafana.nix) and the bearer its metrics datasource presents + # (`swarm-bao-grafana-metrics-bearer`). `create` without `update`, so a stored + # value is never replaced. grafanaOidcReaders = [ { name = "swarm-grafana-oidc"; cn = baoDeploy.grafanaOidcCommonName; policyText = readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/oidc/client" - + createStanza "${credentialMountPath}/data/${grafanaAdminLeaf}"; + + createStanza "${credentialMountPath}/data/${grafanaAdminLeaf}" + + createStanza "${credentialMountPath}/data/${grafanaMetricsBearerLeaf}"; } ]; @@ -982,6 +984,12 @@ let # of `adminPasswordStorePath` in ./swarm-grafana.nix. grafanaAdminLeaf = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/admin/password"; + # The bearer Grafana's metrics datasource presents at the metrics store. + # Grafana's leaf creates it, the store's responder reads it. The nix half of + # `metricsBearerStorePath` in ./swarm-grafana.nix and of `--store-path` in + # ./swarm-victoriametrics.nix. + grafanaMetricsBearerLeaf = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer"; + createStanza = path: '' path "${path}" { capabilities = ["create", "read"] @@ -999,6 +1007,16 @@ let } ]; + # The metrics store's responder, which checks the bearer Grafana presents + # against the one stored for it, and reads nothing else. + metricsAuthReaders = [ + { + name = "swarm-metrics-auth"; + cn = baoDeploy.metricsAuthCommonName; + policyText = readStanza "${credentialMountPath}/data/${grafanaMetricsBearerLeaf}"; + } + ]; + # The CI runner's token fetch. The one reader of the path, which the # controller writes and cannot read back. ciRunnerReaders = [ @@ -1859,6 +1877,19 @@ in ''; }; + metricsAuthCommonName = lib.mkOption { + type = lib.types.str; + default = "swarm-metrics-auth"; + description = '' + Subject the store's `swarm-metrics-auth` cert-auth role accepts: the + identity the metrics store's Grafana auth responder presents to read + the bearer Grafana's datasource sends, + `swarm/services//metrics/bearer`, and nothing else. + + ⚠️ Reserved as a hive name by ./swarm.nix, like its siblings. + ''; + }; + ciRunnerCommonName = lib.mkOption { type = lib.types.str; default = "swarm-ci-runner"; @@ -3274,6 +3305,8 @@ in systemd.services.swarm-bao-logs-auth-policy = readerPolicyUnit "write the log store responder's agent-credential-reader bao policy and cert-auth role" logsAuthReaders; + systemd.services.swarm-bao-metrics-auth-policy = readerPolicyUnit "write the metrics store responder's Grafana-bearer-reader bao policy and cert-auth role" metricsAuthReaders; + systemd.services.swarm-bao-ci-runner-policy = readerPolicyUnit "write the CI runner's registration-token-reader bao policy and cert-auth role" ciRunnerReaders; systemd.services.swarm-bao-review-classifier-policy = readerPolicyUnit "write the review classifier's API-key-reader bao policy and cert-auth role" reviewClassifierReaders; @@ -3408,6 +3441,7 @@ in "swarm-bao-hive-leaf-policy.service" "swarm-bao-ci-runner-policy.service" "swarm-bao-logs-auth-policy.service" + "swarm-bao-metrics-auth-policy.service" "swarm-bao-forge-publisher-policy.service" ] ++ granterAfter; diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index a8e30bf0..84d10961 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -263,6 +263,16 @@ let adminPasswordPath = "/var/lib/grafana-admin/password"; hostAdminPasswordPath = "/var/lib/nixos-containers/${cfg.machine}${adminPasswordPath}"; + # The bearer the metrics datasource presents at the metrics store's query + # API, which `swarm-metrics-auth` on the store's host checks against the + # same store path. Minted once by `swarm-bao-grafana-metrics-bearer` below, + # under the same grant as the admin password. The file holds the whole + # header value, `Bearer `: nixpkgs accepts a `secureJsonData` entry + # only when it is exactly one `$__file{}`. + metricsBearerStorePath = "secret/swarm/services/${cfg.oidc.clientId}/metrics/bearer"; + metricsBearerPath = "/var/lib/grafana-metrics/bearer"; + hostMetricsBearerPath = "/var/lib/nixos-containers/${cfg.machine}${metricsBearerPath}"; + socketPath = "${deployCfg.grafana.socketDir}/grafana.sock"; # Both static NixOS ids, and both checked rather than assumed: nginx has @@ -328,9 +338,12 @@ in description = '' Where the provisioned datasource points. Defaults to the metrics store's swarm name, served by the gateway of whichever host runs the - store, so Grafana and the store need not share a host. The store's - query routes on that vhost take no credential, so the datasource - sends none. + store, so Grafana and the store need not share a host. + + The datasource sends the bearer `swarm-bao-grafana-metrics-bearer` + keeps in the swarm secret store, which is what the store's query + routes on that vhost check. Whatever URL this names receives that + bearer. ''; }; @@ -756,6 +769,86 @@ in ''; }; + # The metrics datasource's bearer: created in the store if it holds none, + # then copied into the container, under Grafana's own leaf. Same shape and + # grant as the admin password above. The store's `swarm-metrics-auth` + # reads the same path, so nothing is placed by hand on either host. + systemd.services.swarm-bao-grafana-metrics-bearer = { + description = "create Grafana's metrics store bearer in the swarm secret store if absent and fetch it"; + # Ordered after the store where it runs here: + # ./glue-bao-readers-store-order.nix. + before = [ "container@${cfg.machine}.service" ]; + wantedBy = [ + "multi-user.target" + "container@${cfg.machine}.service" + ]; + path = [ + baoDeploy.package + pkgs.coreutils + pkgs.openssl + pkgs.systemd + ]; + inherit (storeRetry) startLimitBurst startLimitIntervalSec; + serviceConfig = storeRetry.serviceConfig // { + Type = "oneshot"; + RemainAfterExit = true; + SyslogIdentifier = "swarm-bao-grafana-metrics-bearer"; + TimeoutStartSec = 30; + }; + environment = { + BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; + BAO_CLIENT_CERT = baoDeploy.grafanaOidcClientCertFile; + BAO_CLIENT_KEY = baoDeploy.grafanaOidcClientKeyFile; + } + // lib.optionalAttrs (baoDeploy.serverCaFile != null) { + BAO_CACERT = baoDeploy.serverCaFile; + }; + script = '' + set -euo pipefail + + ${atomicWriteSecret} + ${refreshConsumer} + + err="$(mktemp)" + trap 'rm -f "$err"' EXIT + + if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then + echo "could not log in to swarm-bao with Grafana's certificate; leaving Grafana's metrics bearer file as it is." >&2 + cat "$err" >&2 + exit 1 + fi + export BAO_TOKEN + + # Read, create on a miss, read again: the admin password's sequence + # above, for its reasons. + if ! bearer="$(bao kv get -field=value ${lib.escapeShellArg metricsBearerStorePath} 2>"$err")"; then + minted="$(openssl rand -hex 32)" + if ! printf '%s' "$minted" | bao kv put -cas=0 ${lib.escapeShellArg metricsBearerStorePath} value=- >/dev/null 2>"$err"; then + echo "could not create ${metricsBearerStorePath}; reading it back in case another run created it." >&2 + cat "$err" >&2 + fi + if ! bearer="$(bao kv get -field=value ${lib.escapeShellArg metricsBearerStorePath} 2>"$err")"; then + echo "swarm-bao did not return ${metricsBearerStorePath}; Grafana does not start without it." >&2 + cat "$err" >&2 + exit 1 + fi + fi + if [ -z "$bearer" ]; then + echo "swarm-bao returned an empty ${metricsBearerStorePath}; leaving the file as it is." >&2 + exit 1 + fi + + # Owner and mode as for the OIDC secret above, for its reasons. + install -d -m 0755 ${lib.escapeShellArg (builtins.dirOf hostMetricsBearerPath)} + if secret_differs ${lib.escapeShellArg hostMetricsBearerPath} "Bearer $bearer"; then + atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg hostMetricsBearerPath} "Bearer $bearer" + fi + + # `$__file{}` in a provisioned datasource is read as Grafana starts. + refresh_consumer ${lib.escapeShellArg cfg.machine} grafana.service ${lib.escapeShellArg hostMetricsBearerPath} + ''; + }; + containers.${cfg.machine} = { autoStart = true; ephemeral = false; @@ -1017,6 +1110,10 @@ in url = deployCfg.grafana.datasourceUrl; access = "proxy"; isDefault = true; + # The store's query routes admit this bearer or an operator's + # session, and nothing else. + jsonData.httpHeaderName1 = "Authorization"; + secureJsonData.httpHeaderValue1 = "$__file{${metricsBearerPath}}"; } { name = "VictoriaLogs"; diff --git a/nix/host-modules/swarm-victoriametrics.nix b/nix/host-modules/swarm-victoriametrics.nix index f1f0d969..50bf341f 100644 --- a/nix/host-modules/swarm-victoriametrics.nix +++ b/nix/host-modules/swarm-victoriametrics.nix @@ -36,13 +36,14 @@ let # container, which knows it as `gateway-nginx`. nginxGid = config.ids.gids.nginx; - # The store's read API: the only part of it the vhost serves without a - # credential. Each location's `proxy_pass` carries the path, so the store - # receives the path nginx matched, not the caller's unnormalised spelling. + # The store's read API, which the vhost forwards to a caller holding + # Grafana's bearer or an operator's session. Each location's `proxy_pass` + # carries the path, so the store receives the path nginx matched, not the + # caller's unnormalised spelling. # # Grafana's Prometheus datasource reads the query, series, label, metadata, # buildinfo, exemplar, rules and alerts routes; the store answers the last - # three with fixed placeholders. vmui is the static files under `/vmui/` + # three with fixed placeholders. vmui (`uiLocations` below) is static files # plus the query API and the remaining read-only routes here. Paths read off # the pinned build's `app/vmselect/main.go` and vmui's `src/`. readPaths = [ @@ -64,22 +65,67 @@ let "/expand-with-exprs" "/prettify-query" "/metric-relabel-debug" - "/vmui" - ]; - # The store serves only `/values` under `/api/v1/label/`, and only - # static files and UI config under `/vmui/`. - readPrefixes = [ - "/api/v1/label/" - "/vmui/" ]; + # The store serves only `/values` under `/api/v1/label/`. + readPrefixes = [ "/api/v1/label/" ]; + + # Bare `auth_request`, no login redirect: a machine caller handed + # `error_page 401 =302` follows it and reads authelia's login page as a 200 + # holding no series. `/__metrics_read_auth` picks the check from the + # caller's `Authorization` (`appendHttpConfig` below). + readAuth = '' + auth_request /__metrics_read_auth; + ''; readLocations = let - to = path: { proxyPass = upstream path; }; + to = path: { + proxyPass = upstream path; + extraConfig = readAuth; + }; in lib.listToAttrs ( map (p: lib.nameValuePair "= ${p}" (to p)) readPaths ++ map (p: lib.nameValuePair "^~ ${p}" (to p)) readPrefixes ); + + # The same three lines as `swarmAuthRequest` in ./swarm-victorialogs.nix, + # for the same browser: an operator without a session is sent to log in. + swarmAuthRequest = '' + auth_request /__metrics_authelia; + auth_request_set $target_url $scheme://$http_host$request_uri; + error_page 401 =302 https://${hyperhiveCfg.swarm.authelia.domain}/?rd=$target_url; + ''; + # vmui's static files and UI config. Only a browser loads these, and its + # queries then go to the read locations above with the same session cookie. + uiLocations = { + "= /vmui" = { + proxyPass = upstream "/vmui"; + extraConfig = swarmAuthRequest; + }; + "^~ /vmui/" = { + proxyPass = upstream "/vmui/"; + extraConfig = swarmAuthRequest; + }; + }; + + # The `auth_request` responder for Grafana's bearer (`swarm-metrics-auth`). + # Its socket's directory holds that socket alone and is traverse-only for + # others, the shape `swarm-logs-auth` in ./swarm-victorialogs.nix has. + metricsAuthUnit = "swarm-metrics-auth"; + metricsAuthSocket = "/run/${metricsAuthUnit}/auth.sock"; + metricsAuthCredential = id: "/run/credentials/${metricsAuthUnit}.service/${id}"; + # Credential id → host path of the responder's store identity. + metricsAuthStoreFiles = { + "bao-client.pem" = deployCfg.victoriametrics.authBaoClientCertFile; + "bao-client-key.pem" = deployCfg.victoriametrics.authBaoClientKeyFile; + } + // lib.optionalAttrs (deployCfg.bao.serverCaFile != null) { + "bao-ca.pem" = deployCfg.bao.serverCaFile; + }; + baoCfg = hyperhiveCfg.swarm.bao; + # The path Grafana's host creates the bearer at, under the KV mount. The + # same leaf as `grafanaMetricsBearerLeaf` in ./swarm-bao.nix. + grafanaBearerStorePath = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer"; in { # What the store IS from any hive's point of view, the name it answers on, @@ -113,6 +159,44 @@ in ''; }; + options.services.hyperhive.deploy.victoriametrics.authPackage = lib.mkOption { + type = lib.types.package; + defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-metrics-auth"; + description = '' + The `swarm-metrics-auth` package: the gateway's `auth_request` responder + that checks the bearer Grafana's metrics datasource presents on the + store's query API against the secret store. + ''; + }; + + options.services.hyperhive.deploy.victoriametrics.authBaoClientCertFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-bao-pki/metrics-auth.pem"; + description = '' + Client certificate `swarm-metrics-auth` presents to the swarm's secret + store to read Grafana's metrics bearer. Its subject must be + {option}`services.hyperhive.deploy.bao.metricsAuthCommonName`. + + Required when `deploy.victoriametrics.enable` is set; unset fails at + eval. ./glue-metrics-auth-bao-identity.nix points it at the leaf + ./glue-bao-tls.nix mints on a host that runs the store; on any other + metrics-store host it is a file the operator places. + + A path, never a value. + ''; + }; + + options.services.hyperhive.deploy.victoriametrics.authBaoClientKeyFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-bao-pki/metrics-auth-key.pem"; + description = '' + Private key for {option}`services.hyperhive.deploy.victoriametrics.authBaoClientCertFile`. + A path, never a value. + ''; + }; + config = lib.mkIf deployCfg.victoriametrics.enable { assertions = [ { @@ -128,6 +212,22 @@ in cannot connect and every read, push and scrape fails with 502. ''; } + { + assertion = + deployCfg.victoriametrics.authBaoClientCertFile != null + && deployCfg.victoriametrics.authBaoClientKeyFile != null; + message = '' + services.hyperhive.deploy.victoriametrics.authBaoClientCertFile and + authBaoClientKeyFile must both be set. + + swarm-metrics-auth reads Grafana's metrics bearer from the swarm's + secret store with this certificate. They default to the leaf the + store mints where services.hyperhive.deploy.bao.enable is set on + this host; elsewhere, issue a leaf whose CN is + services.hyperhive.deploy.bao.metricsAuthCommonName and name it in + both options. + ''; + } ]; # The gateway name, inside `deployCfg.victoriametrics.enable` — that @@ -141,77 +241,118 @@ in services.hyperhive.gateway.enable = lib.mkDefault true; services.hyperhive.gateway.dns.enable = lib.mkDefault true; - services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // { - listen = gatewayCfg.lib.listen; - extraConfig = gatewayCfg.lib.securityHeaders; - locations = readLocations // { - # Nothing reaches the store unless a location here names it. The - # store serves writes and admin calls (`/api/v1/import`, - # `/api/v1/admin/tsdb/delete_series`, ...) on the same listener as - # its queries, and authenticates none of them. - "/".return = "403"; + # Which check a read location's `auth_request` gets. `auth_request` takes a + # fixed URI, so it names `/__metrics_read_auth`, which rewrites to the + # target this map picks: a bearer goes to Grafana's check, anything else + # (an operator's session cookie) to authelia. + services.nginx.appendHttpConfig = '' + map $http_authorization $swarm_metrics_read_auth_target { + "~^Bearer " /__metrics_grafana_auth; + default /__metrics_authelia; + } + ''; - # The swarm UI's quick link opens the bare domain. The store's own - # index page links mostly to paths refused above, so send it to vmui. - "= /".return = "302 /vmui/"; + # `removeAttrs`/`forceSSL`: authelia answers a plain-http auth subrequest + # with 400, which `auth_request` cannot read as anything but a broken + # check. Same as ./swarm-victorialogs.nix's vhost. + services.nginx.virtualHosts."${cfg.domain}" = + (builtins.removeAttrs (gatewayCfg.lib.tlsFor cfg.domain) [ "addSSL" ]) + // { + forceSSL = true; + listen = gatewayCfg.lib.listen; + extraConfig = gatewayCfg.lib.securityHeaders; + locations = + readLocations + // uiLocations + // { + # Nothing reaches the store unless a location here names it. The + # store serves writes and admin calls (`/api/v1/import`, + # `/api/v1/admin/tsdb/delete_series`, ...) on the same listener as + # its queries, and authenticates none of them. + "/".return = "403"; - # The swarm collector's ingest route. `=` so it outranks the `/` - # prefix above. - # - # ⚠️ No `error_page 401 =302` here, and its absence is the point: a - # redirect is right for a browser and wrong for a pusher, which would - # follow it and POST its batch at a login page that answers 200 — - # ingest reporting healthy while storing nothing. - "= /opentelemetry/api/v1/push" = { - proxyPass = upstream "/opentelemetry/api/v1/push"; - extraConfig = '' - auth_request /__metrics_push_authz; - ''; - }; + # The swarm UI's quick link opens the bare domain. The store's own + # index page links mostly to paths refused above, so send it to vmui. + "= /".return = "302 /vmui/"; - # The store's own health, for the swarm collector wherever it runs: - # served on the query listener, so no exporter and no extra port. `=` - # for the same reason as the push route, and no `error_page` for the - # same reason: a scraper would parse a login page as metrics. - # - # The collector requests exactly this URL, declared as - # `publishedScrapeTargets.victoriametrics` in ./swarm-otel-service.nix; - # the two must agree, since that string is also the token's audience. - "= /metrics" = { - proxyPass = upstream "/metrics"; - extraConfig = '' - auth_request /__metrics_push_authz; - ''; - }; + # The swarm collector's ingest route. `=` so it outranks the `/` + # prefix above. + # + # ⚠️ No `error_page 401 =302` here, and its absence is the point: a + # redirect is right for a browser and wrong for a pusher, which would + # follow it and POST its batch at a login page that answers 200 — + # ingest reporting healthy while storing nothing. + "= /opentelemetry/api/v1/push" = { + proxyPass = upstream "/opentelemetry/api/v1/push"; + extraConfig = '' + auth_request /__metrics_authelia; + ''; + }; - # The subrequest for both gated locations above. Authelia admits the - # collector's client anywhere on this name and refuses every other - # subject (`metricsStoreRules` in ./swarm-authelia.nix), so the scrape - # needs no subrequest of its own. Same target and header set as the - # sibling log store's, which took them from `swarm-ui.nix` — - # `X-Original-URL` and `X-Original-Method` are what authelia's - # auth-request implementation reads, and the address it compares the - # token's audience against. - "= /__metrics_push_authz" = { - proxyPass = "https://${hyperhiveCfg.swarm.authelia.domain}/api/authz/auth-request"; - # nixpkgs appends its OWN `Host $host` after extraConfig, which - # would override verifiedProxyTo's — see the comment on - # verifiedProxyTo in hive-gateway/vhost-lib.nix. - recommendedProxySettings = false; - extraConfig = '' - internal; - ${gatewayCfg.lib.verifiedProxyTo hyperhiveCfg.swarm.authelia.domain} - proxy_pass_request_body off; - proxy_set_header Content-Length ""; - proxy_set_header X-Original-Method $request_method; - proxy_set_header X-Original-URL $scheme://$http_host$request_uri; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Host $http_host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - ''; - }; + # The store's own health, for the swarm collector wherever it runs: + # served on the query listener, so no exporter and no extra port. `=` + # for the same reason as the push route, and no `error_page` for the + # same reason: a scraper would parse a login page as metrics. + # + # The collector requests exactly this URL, declared as + # `publishedScrapeTargets.victoriametrics` in ./swarm-otel-service.nix; + # the two must agree, since that string is also the token's audience. + "= /metrics" = { + proxyPass = upstream "/metrics"; + extraConfig = '' + auth_request /__metrics_authelia; + ''; + }; + + # `rewrite … last` re-matches locations, so the subrequest lands in + # one of the two internal locations named by the map. + "= /__metrics_read_auth" = { + extraConfig = '' + internal; + rewrite ^ $swarm_metrics_read_auth_target last; + ''; + }; + + # Grafana's bearer, checked by `swarm-metrics-auth`. The + # `Authorization` header rides the subrequest unchanged. No rate limit + # in front: the responder reads the store at most once per its cache + # TTL, whatever the bearers. + "= /__metrics_grafana_auth" = { + proxyPass = "http://unix:${metricsAuthSocket}:/"; + extraConfig = '' + internal; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + ''; + }; + + # The authelia subrequest. For the collector's two locations above it + # admits only the collector's client; for the read and vmui locations + # it admits an operator's session (`metricsStoreRules` in + # ./swarm-authelia.nix). Same target and header set as the sibling log + # store's, which took them from `swarm-ui.nix` — `X-Original-URL` and + # `X-Original-Method` are what authelia's auth-request implementation + # reads, and the address it compares the token's audience against. + "= /__metrics_authelia" = { + proxyPass = "https://${hyperhiveCfg.swarm.authelia.domain}/api/authz/auth-request"; + # nixpkgs appends its OWN `Host $host` after extraConfig, which + # would override verifiedProxyTo's — see the comment on + # verifiedProxyTo in hive-gateway/vhost-lib.nix. + recommendedProxySettings = false; + extraConfig = '' + internal; + ${gatewayCfg.lib.verifiedProxyTo hyperhiveCfg.swarm.authelia.domain} + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-Method $request_method; + proxy_set_header X-Original-URL $scheme://$http_host$request_uri; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + ''; + }; + }; }; - }; # The socket directory, created host-side before the container starts: # nixos-container refuses to start when a bind source is missing. @@ -224,6 +365,67 @@ in "d ${socketDir} 0770 root ${toString nginxGid} - -" ]; + # Grafana's check on the read API. On the host rather than in the store's + # container: it needs nothing of the store, only nginx's reach to its + # socket and its own store identity. + users.users.${metricsAuthUnit} = { + isSystemUser = true; + group = metricsAuthUnit; + description = "swarm metrics store auth_request responder"; + }; + users.groups.${metricsAuthUnit} = { }; + + systemd.services.${metricsAuthUnit} = { + description = "swarm metrics store auth_request responder for Grafana's bearer"; + wantedBy = [ "multi-user.target" ]; + # Where the store's PKI is minted on this host, the leaf is one of its + # files. Ordering only: elsewhere the unit does not exist. + after = [ + "network.target" + "swarm-bao-pki.service" + ]; + serviceConfig = { + ExecStart = lib.concatStringsSep " " [ + "${deployCfg.victoriametrics.authPackage}/bin/swarm-metrics-auth" + "--socket ${metricsAuthSocket}" + # The role ./swarm-bao.nix writes on the store's `cert` mount. + "--store-cert-role swarm-metrics-auth" + "--store-path ${grafanaBearerStorePath}" + ]; + # Paths only: `argv` is world-readable via /proc//cmdline. + LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") metricsAuthStoreFiles; + # ⚠️ A static user, never `DynamicUser`: with `RuntimeDirectoryPreserve` + # systemd puts a dynamic user's runtime directory under + # `/run/private/` (0700 root:root), which nginx cannot traverse. + User = metricsAuthUnit; + Group = metricsAuthUnit; + RuntimeDirectory = metricsAuthUnit; + # Traverse-only for others, so nginx reaches the socket path. + RuntimeDirectoryMode = "0751"; + # The path never vanishes from under a running nginx; the responder + # unlinks a stale socket as it starts. + RuntimeDirectoryPreserve = "yes"; + Restart = "on-failure"; + RestartSec = "5s"; + SyslogIdentifier = metricsAuthUnit; + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = "disconnected"; + PrivateDevices = true; + NoNewPrivileges = true; + RestrictSUIDSGID = true; + RemoveIPC = true; + }; + environment = { + BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; + BAO_CLIENT_CERT = metricsAuthCredential "bao-client.pem"; + BAO_CLIENT_KEY = metricsAuthCredential "bao-client-key.pem"; + } + // lib.optionalAttrs (metricsAuthStoreFiles ? "bao-ca.pem") { + BAO_CACERT = metricsAuthCredential "bao-ca.pem"; + }; + }; + containers.${cfg.machine} = { autoStart = true; ephemeral = false; diff --git a/nix/host-modules/swarm.nix b/nix/host-modules/swarm.nix index ee393eba..90ffbdcc 100644 --- a/nix/host-modules/swarm.nix +++ b/nix/host-modules/swarm.nix @@ -56,6 +56,7 @@ let deployCfg.bao.natsAuthCommonName deployCfg.bao.ciRunnerCommonName deployCfg.bao.logsAuthCommonName + deployCfg.bao.metricsAuthCommonName deployCfg.bao.reviewClassifierCommonName deployCfg.bao.builderCommonName deployCfg.bao.forgePublisherCommonName diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index e6574e7c..592802b3 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -169,7 +169,7 @@ let _: u: (u.environment.BAO_CLIENT_CERT or null) == granterCertFile ) baoGrantWithConsumers.systemd.services; - # The eighteen units that write a `swarm-*` grant, by name, for the discovery + # The nineteen units that write a `swarm-*` grant, by name, for the discovery # control below. grantingUnitNames = [ "swarm-bao-controller-policy" @@ -187,6 +187,7 @@ let "swarm-bao-hive-leaf-policy" "swarm-bao-ci-runner-policy" "swarm-bao-logs-auth-policy" + "swarm-bao-metrics-auth-policy" "swarm-bao-forge-publisher-policy" "swarm-bao-review-classifier-policy" "swarm-bao-builder-policy" @@ -593,10 +594,10 @@ let # both, and the two are separate principals precisely because a # dashboard is not entitled to a collector's credential. # - # The admin password is the one leaf it may create, and without `update`: - # a grant that could replace it would let a second run rotate the - # password Grafana's database already holds. - name = "the Grafana reader reads its own client secret and creates its admin password, and nothing else"; + # The admin password and the metrics bearer are the leaves it may create, + # and without `update`: a grant that could replace one would let a second + # run rotate a value Grafana or the metrics store already holds. + name = "the Grafana reader reads its own client secret and creates its admin password and metrics bearer, and nothing else"; ok = let s = baoGrantHere.systemd.services.swarm-bao-grafana-oidc-policy.script; @@ -613,6 +614,13 @@ let "read" ]; } + { + path = "secret/data/swarm/services/swarm-grafana/metrics/bearer"; + caps = [ + "create" + "read" + ]; + } ] && !(lib.hasInfix "secret/data/swarm/agents" s) && !(lib.hasInfix "secret/data/swarm/hives" s) @@ -731,6 +739,32 @@ let in lib.hasInfix "/logs-auth.pem ]" s && lib.hasInfix "swarm-logs-auth \"\" clientAuth" s; } + { + # Read-only on the one leaf Grafana's host creates, and nothing else: + # this principal checks a presented bearer and does nothing with it. + name = "the metrics store responder's grant is Grafana's metrics bearer, read-only, and nothing else"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-metrics-auth-policy.script; + in + grantsIn s == [ + { + path = "secret/data/swarm/services/swarm-grafana/metrics/bearer"; + caps = [ "read" ]; + } + ] + && lib.hasInfix "auth/cert/certs/swarm-metrics-auth" s + && lib.hasInfix "allowed_common_names=swarm-metrics-auth" s + && lib.hasInfix "token_policies=swarm-metrics-auth" s; + } + { + name = "the PKI unit signs the metrics store responder's leaf under its own subject"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-pki.script; + in + lib.hasInfix "/metrics-auth.pem ]" s && lib.hasInfix "swarm-metrics-auth \"\" clientAuth" s; + } { # 🩸 The half that makes the policies above bind: a policy grants only # through a token that carries it, and a token is minted by a cert-auth @@ -942,24 +976,24 @@ let } { # A store host without the granter's pair writes its grants some other - # way, so none of the eighteen units may exist. Without this arm + # way, so none of the nineteen units may exist. Without this arm # `lib.mkIf haveGranter` could be dropped from any of them and every other # case here would still pass. - name = "without the granter's pair none of the eighteen granting units render"; + name = "without the granter's pair none of the nineteen granting units render"; ok = let s = baoGranterOptOut.systemd.services; in lib.all (unit: !(s ? ${unit})) (grantingUnitNames ++ [ "swarm-bao-granter-role" ]) - # The control: the same store with the pair renders all eighteen. + # The control: the same store with the pair renders all nineteen. && lib.all (unit: baoGrantHere.systemd.services ? ${unit}) grantingUnitNames; } { - # 🩸 What replaced the silent skip. With no bootstrap token the eighteen still + # 🩸 What replaced the silent skip. With no bootstrap token the nineteen still # render, and a refused granter fails them with the step that fixes it. # A store host that never named a token is told to name one, since the # unit that sets the granter up renders only where it has. - name = "a store host without a bootstrap token renders the eighteen, each failing loudly with the one-time step"; + name = "a store host without a bootstrap token renders the nineteen, each failing loudly with the one-time step"; ok = let s = baoGranterNoToken.systemd.services; @@ -1049,6 +1083,7 @@ let "swarm-services-cert" "swarm-bao-forwarder-oidc" "swarm-bao-grafana-admin" + "swarm-bao-grafana-metrics-bearer" ] ++ policyReaders ); @@ -1776,8 +1811,8 @@ let } { # What makes the case above mean something: discovery by the granter's - # certificate reaches all eighteen units, and each yields calls. - name = "the granter-policy check sees all eighteen granting units, and parses calls from each"; + # certificate reaches all nineteen units, and each yields calls. + name = "the granter-policy check sees all nineteen granting units, and parses calls from each"; ok = lib.sort lib.lessThan (lib.attrNames granterUnits) == lib.sort lib.lessThan grantingUnitNames && lib.all (u: baoCalls u.script != [ ]) (lib.attrValues granterUnits) diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index 986aece1..f9fe80bb 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -478,36 +478,94 @@ let && lib.hasInfix "error_page" browser; } { - # The store authenticates nothing itself, so every location that - # forwards without `auth_request` is open to anything that can reach the - # domain. The query location is the positive control: it shows the - # filter finds forwarding locations at all. - name = "the metrics vhost forwards only read paths without a credential"; + # The store authenticates nothing itself, so a location that forwards to + # it without `auth_request` is open to anything that can reach the + # domain. The query location is the control: it shows the filter finds + # store-forwarding locations at all. + name = "no location on the metrics vhost forwards to the store without auth_request"; ok = let locs = allLocal.services.nginx.virtualHosts."metrics.t.local".locations; - open = lib.filterAttrs ( - _: l: - l.proxyPass != null - && !(lib.hasInfix "auth_request" l.extraConfig) - && !(lib.hasInfix "internal;" l.extraConfig) + toStore = lib.filterAttrs ( + _: l: l.proxyPass != null && lib.hasPrefix "http://unix:/run/swarm-victoriametrics/" l.proxyPass ) locs; - writeWords = [ - "import" - "write" - "admin" - "delete" - "insert" - "internal" - "push" - "prometheus" - "influx" - ]; + open = lib.filterAttrs (_: l: !(lib.hasInfix "auth_request " l.extraConfig)) toStore; in - locs."/".proxyPass == null - && locs."/".return == "403" - && open ? "= /api/v1/query" - && lib.all (n: n != "/" && !(lib.any (w: lib.hasInfix w n) writeWords)) (lib.attrNames open); + toStore ? "= /api/v1/query" + && open == { } + && locs."/".proxyPass == null + && locs."/".return == "403"; + } + { + # The read API's check, link by link: every read location names the + # switch with no login redirect (a machine reader would get login HTML + # as a 200), the switch rewrites to the map's pick, a bearer reaches + # the responder's socket, which the unit binds, and anything else goes + # to authelia. vmui is a browser's, so it redirects to the login. + name = "the metrics read API sends a bearer to swarm-metrics-auth and a session to authelia"; + ok = + let + l = allLocal.services.nginx.virtualHosts."metrics.t.local".locations; + http = allLocal.services.nginx.appendHttpConfig; + unit = allLocal.systemd.services.swarm-metrics-auth; + reads = lib.filterAttrs (n: _: lib.hasPrefix "= /api/v1/" n || lib.hasPrefix "^~ /api/v1/" n) l; + in + reads ? "= /api/v1/query_range" + && lib.all ( + r: + lib.hasInfix "auth_request /__metrics_read_auth;" r.extraConfig + && !(lib.hasInfix "error_page" r.extraConfig) + ) (lib.attrValues reads) + && lib.hasInfix "map \$http_authorization \$swarm_metrics_read_auth_target" http + && lib.hasInfix "\"~^Bearer \" /__metrics_grafana_auth;" http + && lib.hasInfix "default /__metrics_authelia;" http + && + lib.hasInfix "rewrite ^ \$swarm_metrics_read_auth_target last;" + l."= /__metrics_read_auth".extraConfig + && lib.hasInfix "internal;" l."= /__metrics_grafana_auth".extraConfig + && l."= /__metrics_grafana_auth".proxyPass == "http://unix:/run/swarm-metrics-auth/auth.sock:/" + && lib.hasInfix "--socket /run/swarm-metrics-auth/auth.sock" unit.serviceConfig.ExecStart + && lib.hasInfix "auth_request /__metrics_authelia;" l."^~ /vmui/".extraConfig + && lib.hasInfix "error_page 401 =302" l."^~ /vmui/".extraConfig + && l."= /__metrics_authelia".proxyPass == "https://auth.t.local/api/authz/auth-request"; + } + { + # Grafana's host creates the bearer and its datasource sends it; the + # store's responder reads it. Three modules spell the path, and the + # store's grants name it twice: if any one drifts, Grafana sends a value + # the responder never reads, or one of them is refused by the store. + name = "Grafana's datasource sends the bearer swarm-metrics-auth checks, at one store path"; + ok = + let + leaf = "swarm/services/${allLocal.services.hyperhive.swarm.grafana.oidc.clientId}/metrics/bearer"; + ds = + lib.findFirst (d: d.uid == "swarm-victoriametrics") null + allLocal.containers.swarm-grafana.config.services.grafana.provision.datasources.settings.datasources; + bearerUnit = allLocal.systemd.services.swarm-bao-grafana-metrics-bearer; + s = allLocal.systemd.services; + in + ds.jsonData.httpHeaderName1 == "Authorization" + && ds.secureJsonData.httpHeaderValue1 == "$__file{/var/lib/grafana-metrics/bearer}" + && lib.hasInfix "bao kv put -cas=0 secret/${leaf} value=-" bearerUnit.script + && lib.hasInfix "/var/lib/nixos-containers/swarm-grafana/var/lib/grafana-metrics/bearer \"Bearer $bearer\"" bearerUnit.script + && lib.hasInfix "--store-path ${leaf}" s.swarm-metrics-auth.serviceConfig.ExecStart + && lib.hasInfix "path \"secret/data/${leaf}\" {\n capabilities = [\"read\"]\n}" s.swarm-bao-metrics-auth-policy.script + && lib.hasInfix "path \"secret/data/${leaf}\" {\n capabilities = [\"create\", \"read\"]\n}" s.swarm-bao-grafana-oidc-policy.script; + } + { + # The responder reads the store as itself, with the leaf the store's + # host mints for it. + name = "swarm-metrics-auth presents its own store leaf"; + ok = + let + v = allLocal.services.hyperhive.deploy.victoriametrics; + lc = allLocal.systemd.services.swarm-metrics-auth.serviceConfig.LoadCredential; + in + lib.hasSuffix "/metrics-auth.pem" v.authBaoClientCertFile + && lib.hasSuffix "/metrics-auth-key.pem" v.authBaoClientKeyFile + && builtins.elem "bao-client.pem:${v.authBaoClientCertFile}" lc + && builtins.elem "bao-client-key.pem:${v.authBaoClientKeyFile}" lc + && lib.hasInfix "/metrics-auth.pem ]" allLocal.systemd.services.swarm-bao-pki.script; } { # The read counterpart to the ingest location: an agent queries the log @@ -592,7 +650,7 @@ let # `/run/private/` (0700 root:root), which nginx cannot traverse, so a # socket there answers every proxied request with a 502. Each `unix:` # upstream under `/run//` is matched to the units creating ``. - # The first clause is the control: the match finds swarm-logs-auth's. + # The first two clauses are the control: the match finds both responders. name = "no unix socket nginx proxies to sits in a DynamicUser's preserved runtime directory"; ok = let @@ -626,7 +684,9 @@ let "no" ]); in - owners ? swarm-logs-auth && !(lib.any (u: hidden u.serviceConfig) (lib.attrValues owners)); + owners ? swarm-logs-auth + && owners ? swarm-metrics-auth + && !(lib.any (u: hidden u.serviceConfig) (lib.attrValues owners)); } { # Every agent bearer the responder has not recently granted costs a diff --git a/nix/module-eval/swarm-otel-identity.nix b/nix/module-eval/swarm-otel-identity.nix index 2d27d63c..45e8f1ac 100644 --- a/nix/module-eval/swarm-otel-identity.nix +++ b/nix/module-eval/swarm-otel-identity.nix @@ -287,20 +287,21 @@ let in m != null && m.proxyPass == "http://unix:/run/swarm-victoriametrics/victoriametrics.sock:/metrics" - && lib.hasInfix "auth_request /__metrics_push_authz;" m.extraConfig + && lib.hasInfix "auth_request /__metrics_authelia;" m.extraConfig && !(lib.hasInfix "error_page" m.extraConfig) - && lib.hasInfix "internal;" locs."= /__metrics_push_authz".extraConfig + && lib.hasInfix "internal;" locs."= /__metrics_authelia".extraConfig && locs."/".return == "403" && locs."/".proxyPass == null; } { - # The metrics store's name admits the collector's client on every path - # and refuses every other subject. The rules carry no `resources`, so no - # path or query spelling misses them and reaches `default_policy`, and - # they come before the apex rule. That is only right while the push - # route and `/metrics` are the vhost's sole authelia-gated locations, - # which the last clause pins. - name = "authelia admits only the collector anywhere on the metrics store's name"; + # The metrics store's name admits the collector's client on every path, + # an operator on every path but the collector's two, and nobody else. + # Only the collector's two paths carry `resources`, as prefixes, so no + # path or query spelling of them reaches the operator rule, and no other + # spelling reaches `default_policy`. The rules come before the apex + # rule. The collector's two locations call authelia directly, so a + # bearer there never reaches Grafana's check. + name = "authelia admits the collector anywhere on the metrics store's name and operators only off its two paths"; ok = let h = allLocal.services.hyperhive; @@ -311,38 +312,33 @@ let storeRules = lib.filter (r: r.domain == "metrics.t.local") indexed; apex = lib.findFirst (r: r.domain == "t.local") null indexed; policy = subject: autheliaPolicy allLocal subject "metrics.t.local"; - paths = [ + collectorPaths = [ "/opentelemetry/api/v1/push" + "/opentelemetry/api/v1/push?x=1" "/metrics" "/metrics?x=1" + ]; + readPaths = [ "/api/v1/query" + "/api/v1/query_range?query=up" + "/vmui/" ]; locs = allLocal.services.nginx.virtualHosts.${h.swarm.victoriametrics.domain}.locations; in - map (r: removeAttrs r [ "i" ]) storeRules == [ - { - domain = "metrics.t.local"; - policy = "one_factor"; - subject = [ collector ]; - } - { + lib.length storeRules == 4 + && + removeAttrs (lib.last storeRules) [ "i" ] == { domain = "metrics.t.local"; policy = "deny"; } - ] && apex != null && lib.all (r: r.i < apex.i) storeRules - && lib.all (p: policy collector p == "one_factor") paths - && lib.all (p: policy "user:someone" p == "deny") paths - && lib.all (p: policy "group:admins" p == "deny") paths - && - lib.hasInfix "auth_request /__metrics_push_authz;" - locs."= /opentelemetry/api/v1/push".extraConfig - && - lib.attrNames (lib.filterAttrs (_: l: lib.hasInfix "auth_request" (l.extraConfig or "")) locs) == [ - "= /metrics" - "= /opentelemetry/api/v1/push" - ]; + && lib.all (p: policy collector p == "one_factor") (collectorPaths ++ readPaths) + && lib.all (p: policy "user:someone" p == "deny") (collectorPaths ++ readPaths) + && lib.all (p: policy "group:admins" p == "deny") collectorPaths + && lib.all (p: policy "group:admins" p == "one_factor") readPaths + && lib.hasInfix "auth_request /__metrics_authelia;" locs."= /opentelemetry/api/v1/push".extraConfig + && lib.hasInfix "auth_request /__metrics_authelia;" locs."= /metrics".extraConfig; } { # Every published target admits the collector's client and refuses any @@ -398,7 +394,9 @@ let let rules = autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules; - metricsRules = lib.filter (r: r ? resources) rules; + # The metrics store's own name is the case above's; its rules carry + # `resources` for its push route and `/metrics` together. + metricsRules = lib.filter (r: r ? resources && r.domain != "metrics.t.local") rules; re = lib.head (lib.head metricsRules).resources; table = [ { diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix index 95ae8613..b74aab88 100644 --- a/nix/module-eval/swarm-services-switch.nix +++ b/nix/module-eval/swarm-services-switch.nix @@ -540,7 +540,7 @@ let { # No interface but loopback, so the socket is the store's only listener # the host can reach. Every location that forwards to the store goes - # through it; the authelia subrequest is the one `https://` upstream. + # through it; the internal auth subrequests are the only other upstreams. # The read, push and scrape locations are named so an empty filter # can't pass. name = "the metrics store runs on a netns of its own and the gateway reaches it over its socket"; @@ -553,8 +553,10 @@ let viaSock = path: "http://unix:${sock}:${path}"; locs = host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.victoriametrics.domain}.locations; - toStore = lib.filter (p: p != null && !(lib.hasPrefix "https://" p)) ( - map (l: l.proxyPass) (lib.attrValues locs) + toStore = map (l: l.proxyPass) ( + lib.filter (l: l.proxyPass != null && !(lib.hasInfix "internal;" l.extraConfig)) ( + lib.attrValues locs + ) ); unit = c.config.systemd.services.victoriametrics; nginxGid = host.ids.gids.nginx; @@ -749,6 +751,7 @@ let "swarm-bao-hive-otel-oidc" "swarm-bao-grafana-oidc" "swarm-bao-grafana-admin" + "swarm-bao-grafana-metrics-bearer" "swarm-bao-nats-callout-xkey" "swarm-bao-ci-runner" "swarm-bao-otel-oidc" @@ -771,7 +774,7 @@ let !(lib.any (n: lib.elem n (s.after ++ s.wants ++ s.requires)) storeUnits); in lib.all waits readers - && lib.length readersElsewhere == 6 + && lib.length readersElsewhere == 7 && lib.all namesNoStoreUnit readersElsewhere && !(lib.any (u: storeOnlyNoHive.systemd.services ? ${u}) readers); } diff --git a/nix/packages/default.nix b/nix/packages/default.nix index bfcafd6b..a69602bb 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -167,6 +167,10 @@ in # `daemonBins` like the responder above: it runs on the log store's host only. swarm-logs-auth = mkBinPackage "swarm-logs-auth" "hyperhive swarm log store agent auth responder"; + # The metrics store gateway's `auth_request` responder for Grafana's bearer. Out + # of `daemonBins` like the two above: it runs on the metrics store's host only. + swarm-metrics-auth = mkBinPackage "swarm-metrics-auth" "hyperhive swarm metrics store Grafana auth responder"; + # The matrix container's control binary; `mint` publishes the sender token. Out of `daemonBins` for # "runs *inside* a container, not on the host" reason as the responder # above, and with a second one: putting it in the core bundle would place diff --git a/swarm-metrics-auth/Cargo.toml b/swarm-metrics-auth/Cargo.toml new file mode 100644 index 00000000..6d90939f --- /dev/null +++ b/swarm-metrics-auth/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "swarm-metrics-auth" +version.workspace = true +edition.workspace = true + +[[bin]] +name = "swarm-metrics-auth" +path = "src/main.rs" + +[dependencies] +anyhow.workspace = true +axum.workspace = true +clap.workspace = true +serde.workspace = true +# Comparing the presented bearer with the stored one. +subtle.workspace = true +# Only for the rustls provider the store client needs. +swarm-queue-client.workspace = true +swarm-secret-client.workspace = true +tokio.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true + +[lints] +workspace = true diff --git a/swarm-metrics-auth/src/main.rs b/swarm-metrics-auth/src/main.rs new file mode 100644 index 00000000..41076d78 --- /dev/null +++ b/swarm-metrics-auth/src/main.rs @@ -0,0 +1,354 @@ +//! The metrics store gateway's `auth_request` responder for Grafana. +//! +//! nginx sends it every request to the store's query API that carries a +//! `Bearer` (`swarm-victoriametrics.nix`). It answers 200 when the bearer is +//! the value stored at `--store-path`, which Grafana's host creates and its +//! datasource presents, and 401 otherwise, store failures included: +//! `auth_request` turns any other status into a 500 for the caller. +//! +//! It reads the store under its own certificate, whose policy reaches that one +//! path. It serves a unix socket whose directory only the gateway's nginx may +//! enter. +//! +//! The stored value is kept for `--cache-ttl` seconds, and no request reaches +//! the store within that window: a replaced bearer is refused, and the one it +//! replaced still accepted, for up to that long. Lookups are serialised, so a +//! burst of requests against an expired copy costs one. + +use std::future::Future; +use std::os::unix::fs::PermissionsExt as _; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use anyhow::Context as _; +use axum::extract::State; +use axum::http::{HeaderMap, StatusCode, header}; +use clap::Parser; +use subtle::ConstantTimeEq as _; +use swarm_secret_client::client::{DEFAULT_CERT_MOUNT, Settings}; + +/// How long one store lookup may take before the request is denied. Well +/// under nginx's 60s `proxy_read_timeout`, so a slow store reads as a 401 +/// rather than a gateway timeout. +const LOOKUP_TIMEOUT: Duration = Duration::from_secs(5); + +#[derive(Parser)] +#[command(about = "auth_request responder: checks Grafana's metrics bearer against the store")] +struct Args { + /// Unix socket to serve on. + #[arg(long)] + socket: PathBuf, + /// Cert-auth role this responder logs in to the store as. + #[arg(long, default_value = "swarm-metrics-auth")] + store_cert_role: String, + /// KV path, under the `secret` mount, holding the bearer in its `value` + /// field. + #[arg(long)] + store_path: String, + /// Seconds the stored bearer is used without reading the store again; + /// 0 reads it on every request. + #[arg(long, default_value_t = 60)] + cache_ttl: u64, +} + +/// Where the stored bearer comes from. +trait BearerSource { + /// The stored bearer, `None` when nothing is stored, or an error when the + /// store could not answer. + fn lookup(&self) -> impl Future>> + Send; +} + +/// The object at the bearer's path. `bao kv put … value=-` on Grafana's host +/// writes this field. +#[derive(serde::Deserialize)] +struct Stored { + value: String, +} + +/// The swarm secret store, reached with the responder's own certificate. +struct Store { + settings: Settings, + cert_role: String, + path: String, +} + +impl BearerSource for Store { + /// Logs in per lookup, so no token is left to expire inside a long-lived + /// process. + async fn lookup(&self) -> anyhow::Result> { + let store = swarm_secret_client::SecretStore::connect( + &self.settings, + &self.cert_role, + DEFAULT_CERT_MOUNT, + ) + .await + .context("logging in to the secret store")?; + let stored: Option = store + .read_optional(&self.path) + .await + .with_context(|| format!("reading {}", self.path))?; + Ok(stored.map(|s| s.value)) + } +} + +struct Cached { + value: String, + read_at: Instant, +} + +struct Responder { + source: S, + ttl: Duration, + cached: tokio::sync::Mutex>, +} + +/// The answer for one subrequest's `Authorization` header, at `now`. +async fn decide( + responder: &Responder, + authorization: Option<&str>, + now: Instant, +) -> StatusCode { + let Some(bearer) = authorization.and_then(|v| v.strip_prefix("Bearer ")) else { + tracing::warn!("no bearer token; denying"); + return StatusCode::UNAUTHORIZED; + }; + // Held across the lookup, so concurrent requests wait for one read + // rather than each starting their own. + let mut cached = responder.cached.lock().await; + let fresh = cached + .as_ref() + .is_some_and(|c| now.saturating_duration_since(c.read_at) < responder.ttl); + if !fresh { + match tokio::time::timeout(LOOKUP_TIMEOUT, responder.source.lookup()).await { + Ok(Ok(Some(value))) if !value.is_empty() => { + *cached = Some(Cached { + value, + read_at: now, + }); + } + Ok(Ok(_)) => { + tracing::warn!("no bearer is stored; denying"); + return StatusCode::UNAUTHORIZED; + } + Ok(Err(e)) => { + tracing::warn!(error = format!("{e:#}"), "bearer lookup failed; denying"); + return StatusCode::UNAUTHORIZED; + } + Err(_) => { + tracing::warn!("bearer lookup timed out; denying"); + return StatusCode::UNAUTHORIZED; + } + } + } + let Some(stored) = cached.as_ref() else { + return StatusCode::UNAUTHORIZED; + }; + // Constant time, so the time to refuse says nothing about how much of the + // bearer was right. + if bool::from(stored.value.as_bytes().ct_eq(bearer.as_bytes())) { + StatusCode::OK + } else { + tracing::warn!("bearer does not match the stored one; denying"); + StatusCode::UNAUTHORIZED + } +} + +async fn handle(State(responder): State>>, headers: HeaderMap) -> StatusCode { + let authorization = headers + .get(header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()); + decide(&responder, authorization, Instant::now()).await +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + swarm_queue_client::install_crypto_provider(); + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + // stdout is journald, which keeps ANSI escapes as raw bytes. + .with_ansi(false) + .init(); + let args = Args::parse(); + + let settings = Settings::from_env() + .context("reading the secret store's address and this responder's identity")?; + let responder = Arc::new(Responder { + source: Store { + settings, + cert_role: args.store_cert_role, + path: args.store_path, + }, + ttl: Duration::from_secs(args.cache_ttl), + cached: tokio::sync::Mutex::new(None), + }); + + // `RuntimeDirectoryPreserve=yes` can leave the previous run's socket; the + // directory is this unit's alone, so unlinking it is safe. + if let Err(e) = std::fs::remove_file(&args.socket) + && e.kind() != std::io::ErrorKind::NotFound + { + return Err(e) + .with_context(|| format!("clearing stale socket at {}", args.socket.display())); + } + let listener = tokio::net::UnixListener::bind(&args.socket) + .with_context(|| format!("binding {}", args.socket.display()))?; + // nginx is another user and `connect(2)` needs write. The directory is + // the access control, not this mode. + std::fs::set_permissions(&args.socket, std::fs::Permissions::from_mode(0o666)) + .with_context(|| format!("chmod {}", args.socket.display()))?; + tracing::info!(socket = %args.socket.display(), "swarm-metrics-auth listening"); + + let app = axum::Router::new().fallback(handle).with_state(responder); + axum::serve(listener, app).await.context("serving") +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex; + use std::sync::atomic::{AtomicUsize, Ordering}; + + /// A store answering `stored` (or failing when it is `Err`), counting + /// lookups. + struct Fake { + stored: Mutex, ()>>, + lookups: AtomicUsize, + } + + impl Fake { + fn holding(value: &str) -> Self { + Self { + stored: Mutex::new(Ok(Some(value.to_owned()))), + lookups: AtomicUsize::new(0), + } + } + + fn set(&self, stored: Result, ()>) { + *self.stored.lock().unwrap() = stored.map(|v| v.map(str::to_owned)); + } + + fn lookups(&self) -> usize { + self.lookups.load(Ordering::SeqCst) + } + } + + impl BearerSource for &Fake { + async fn lookup(&self) -> anyhow::Result> { + self.lookups.fetch_add(1, Ordering::SeqCst); + self.stored + .lock() + .unwrap() + .clone() + .map_err(|()| anyhow::anyhow!("store unreachable")) + } + } + + const TTL: Duration = Duration::from_mins(1); + const SECRET: &str = "0f1e2d3c4b5a"; + const GOOD: &str = "Bearer 0f1e2d3c4b5a"; + + fn responder(store: &Fake) -> Responder<&Fake> { + Responder { + source: store, + ttl: TTL, + cached: tokio::sync::Mutex::new(None), + } + } + + #[tokio::test] + async fn the_stored_bearer_is_200() { + let store = Fake::holding(SECRET); + assert_eq!( + decide(&responder(&store), Some(GOOD), Instant::now()).await, + StatusCode::OK + ); + } + + #[tokio::test] + async fn everything_else_is_401() { + let store = Fake::holding(SECRET); + let r = responder(&store); + for authorization in [ + None, + Some(SECRET), + Some("Basic 0f1e2d3c4b5a"), + Some("Bearer 0f1e2d3c4b5b"), + Some("Bearer 0f1e2d3c4b5"), + Some("Bearer "), + Some("Bearer authelia_at_abc.def"), + ] { + assert_eq!( + decide(&r, authorization, Instant::now()).await, + StatusCode::UNAUTHORIZED, + "{authorization:?}" + ); + } + } + + /// Nothing stored, an empty value, and a store that cannot answer are + /// all denials, whatever the bearer. + #[tokio::test] + async fn no_usable_stored_value_is_401() { + let store = Fake::holding(SECRET); + let r = responder(&store); + for stored in [Ok(None), Ok(Some("")), Err(())] { + store.set(stored); + assert_eq!( + decide(&r, Some(GOOD), Instant::now()).await, + StatusCode::UNAUTHORIZED, + "{stored:?}" + ); + assert_eq!( + decide(&r, Some("Bearer "), Instant::now()).await, + StatusCode::UNAUTHORIZED, + "{stored:?}" + ); + } + } + + /// Within the TTL no request reaches the store, a wrong bearer included, + /// and a replacement the store already holds goes unseen; after it the + /// store is read again. + #[tokio::test] + async fn the_stored_value_is_reused_until_the_ttl() { + let store = Fake::holding(SECRET); + let r = responder(&store); + let t0 = Instant::now(); + assert_eq!(decide(&r, Some(GOOD), t0).await, StatusCode::OK); + store.set(Ok(Some("replaced"))); + let within = t0 + Duration::from_secs(59); + assert_eq!(decide(&r, Some(GOOD), within).await, StatusCode::OK); + assert_eq!( + decide(&r, Some("Bearer replaced"), within).await, + StatusCode::UNAUTHORIZED + ); + assert_eq!(store.lookups(), 1); + assert_eq!( + decide(&r, Some(GOOD), t0 + TTL).await, + StatusCode::UNAUTHORIZED + ); + assert_eq!( + decide(&r, Some("Bearer replaced"), t0 + TTL).await, + StatusCode::OK + ); + assert_eq!(store.lookups(), 2); + } + + /// A failed read leaves nothing usable cached, so the next request reads + /// the store again rather than waiting out a TTL. + #[tokio::test] + async fn a_failed_lookup_is_retried_on_the_next_request() { + let store = Fake::holding(SECRET); + store.set(Err(())); + let r = responder(&store); + let t0 = Instant::now(); + assert_eq!(decide(&r, Some(GOOD), t0).await, StatusCode::UNAUTHORIZED); + store.set(Ok(Some(SECRET))); + assert_eq!(decide(&r, Some(GOOD), t0).await, StatusCode::OK); + assert_eq!(store.lookups(), 2); + } +}