From 6aab2ebb65cf4ac6f69f9314da6b8ee5442b8961 Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 14 Oct 2026 19:53:36 +0200 Subject: [PATCH] swarm metrics store: the read API needs Grafana's bearer or an operator session The metrics vhost forwarded the store's read API (query, series, labels, export, vmui, ...) with no credential, so anything that could reach metrics. could read every hive's and agent's metrics. mara ruled on checked by an auth_request responder, and operators use their authelia session. - swarm-metrics-auth (new crate): auth_request responder on the metrics store's host. Compares the presented bearer, in constant time, with the one stored at swarm/services//metrics/bearer, read under its own bao leaf and cached for 60s. Static user, the post-#5264 swarm-logs-auth unit shape. - swarm-bao-grafana-metrics-bearer: on Grafana's host, creates that bearer in bao if absent (create without update, Grafana's existing leaf) and writes "Bearer " into the container; the datasource sends it as its Authorization header via $__file{}. - The vhost: read locations take auth_request through a map (a bearer goes to swarm-metrics-auth, anything else to authelia, no login redirect); vmui takes an authelia session with the login redirect; forceSSL like the logs vhost. The authelia subrequest location is renamed from /__metrics_push_authz to /__metrics_authelia since it now serves reads. - Authelia: the collector keeps every path; everyone else is refused the collector's push and /metrics paths; group admins is admitted to the rest; then deny. - bao: a swarm-metrics-auth cert role and policy (read on the one leaf), its leaf minted by glue-bao-tls, the CN reserved as a hive name, and Grafana's grant gains create on the bearer leaf. Fixes #3596 --- Cargo.lock | 16 + Cargo.toml | 1 + docs/process/gotchas.md | 3 +- docs/swarm/credentials.md | 1 + docs/swarm/secrets.md | 48 +-- docs/swarm/services.md | 31 +- flake.nix | 3 + nix/host-modules/default.nix | 1 + .../glue-bao-readers-policy-order.nix | 6 +- .../glue-bao-readers-store-order.nix | 1 + nix/host-modules/glue-bao-tls.nix | 7 + .../glue-metrics-auth-bao-identity.nix | 32 ++ nix/host-modules/swarm-authelia.nix | 43 ++- nix/host-modules/swarm-bao.nix | 40 +- nix/host-modules/swarm-grafana.nix | 103 ++++- nix/host-modules/swarm-victoriametrics.nix | 358 ++++++++++++++---- nix/host-modules/swarm.nix | 1 + nix/module-eval/bao-grants.nix | 59 ++- nix/module-eval/core-toggle.nix | 114 ++++-- nix/module-eval/swarm-otel-identity.nix | 58 ++- nix/module-eval/swarm-services-switch.nix | 11 +- nix/packages/default.nix | 4 + swarm-metrics-auth/Cargo.toml | 25 ++ swarm-metrics-auth/src/main.rs | 354 +++++++++++++++++ 24 files changed, 1123 insertions(+), 197 deletions(-) create mode 100644 nix/host-modules/glue-metrics-auth-bao-identity.nix create mode 100644 swarm-metrics-auth/Cargo.toml create mode 100644 swarm-metrics-auth/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 4a496e0b..592d785a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4943,6 +4943,22 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "swarm-metrics-auth" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "clap", + "serde", + "subtle", + "swarm-queue-client", + "swarm-secret-client", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "swarm-nats-auth" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 00087010..096d4bd8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ members = [ "swarm-review-classifier", "swarm-logs", "swarm-logs-auth", + "swarm-metrics-auth", "swarm-secret-client", "swarmctl", ] diff --git a/docs/process/gotchas.md b/docs/process/gotchas.md index 55a1c546..e2ee26f2 100644 --- a/docs/process/gotchas.md +++ b/docs/process/gotchas.md @@ -82,7 +82,8 @@ With both set, systemd creates the runtime directory under `/run/private/` `RuntimeDirectoryPreserve=`). No unprivileged user can traverse that, so nginx's `connect()` to a socket in it fails with `EACCES` whatever the directory and socket modes are. A unit serving a socket to another user -runs as a static user instead (`swarm-controller`, `swarm-logs-auth`). +runs as a static user instead (`swarm-controller`, `swarm-logs-auth`, +`swarm-metrics-auth`). `module-eval-core-toggle` fails on any `unix:` upstream of nginx owned by such a unit. diff --git a/docs/swarm/credentials.md b/docs/swarm/credentials.md index ba520a0f..7fd613a3 100644 --- a/docs/swarm/credentials.md +++ b/docs/swarm/credentials.md @@ -74,6 +74,7 @@ of the cell says how. | `swarm/hives//oidc/client` | authelia, at its first boot; `swarm-secret-publish` copies it in | hive-c0re, under the hive's own certificate, when it first connects to the queue | ❌ authelia mints it once. A re-mint is republished by `swarm-secret-publish`'s path unit | ✅ read until the first connect succeeds, retried with a backoff of up to five minutes while it's missing; then held in memory and read again when authelia refuses it | | `swarm/services//oidc/client` | authelia | that service's own reader, under its own certificate — the queue responder's under `swarm-nats-auth`'s, the homeserver's under `swarm-matrix-ctl`'s, the forge's under `swarm-forge-publisher`'s, the review classifier's under `swarm-review-classifier`'s; the bao UI's by the store's granter | must be stated | must be stated | | `swarm/services/ci-runner/registration-token` | `swarm-controller`, from the forge's admin API, at start and in a five-minute pass; its grant on the path is write-only | `swarm-bao-ci-runner` on the runner's host, under the runner's own certificate, into a file only `swarm-ci-runner-register` in the runner's container loads; no other principal's policy reads it | ❌ the forge hands out one token until an admin resets it; the controller writes a reset token within five minutes | ✅ `swarm-bao-ci-runner` reads it at boot, on deploy and every ten minutes, and restarts the runner when the value changed | +| `swarm/services//metrics/bearer` | `swarm-bao-grafana-metrics-bearer` on Grafana's host, under Grafana's certificate, when the store holds none | `swarm-metrics-auth` on the metrics store's host, under its own certificate, at most once a minute, to check the bearer Grafana's datasource presents at the store's read API | ❌ nothing re-mints it; an operator deletes it and restarts `swarm-bao-grafana-metrics-bearer` | ✅ `swarm-metrics-auth` reads it again once its copy is a minute old. ❌ Grafana reads its file copy only as it starts; the bearer unit restarts Grafana when the copy changes | | `swarm/services/review-classifier/anthropic-api-key` | the operator, by hand in the store's UI, field `api_key` | `swarm-review-classifier` in its container, under its own certificate, at the start of every API key run; held for that run and handed to `claude`'s environment only; no other principal's policy reads it | ❌ an operator's key; replaced by writing a new one | ✅ read on every run, so a written or removed key takes effect on the next run | | `swarm/services/review-classifier/acp-provider` | the operator, by hand in the store's UI, field `api_key` | `swarm-review-classifier` in its container, under its own certificate, at the start of every ACP run; held for that run and handed to `opencode`'s environment only; no other principal's policy reads it | ❌ an operator's key; replaced by writing a new one | ✅ read on every run, so a written or removed key takes effect on the next run | | `swarm/services/review-classifier/forge/token` | `swarm-controller`, with the forge's admin API, for its `review-classifier` account, at start and in a five-minute pass; the secret publisher's policy denies it | `swarm-controller` under its own certificate, before it decides whether to mint, and on every review it posts as the account; the classifier service's policy doesn't read it | ✅ the controller's pass re-mints when the stored token is missing or no longer matches the forge (last eight characters and scopes) | ✅ the controller reads the store on every review it posts | diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index 48bbba74..0c590ac2 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -43,29 +43,31 @@ neither is a renaming of the other. ## Swarm-level — one of each per swarm -| secret | generated by | lives at | hive elsewhere | -| -------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| swarm-services root CA (cert + key) | `swarm-bao.nix`'s granting unit, inside the store's `pki` mount | the **key** never leaves the mount; the **cert** lands at `/var/lib/swarm-bao-services-pki/services-root.pem` (0644) and at `/var/lib/hive-tls/swarm-services-root.pem` on every hive | nothing to copy — every hive receives the cert with the leaf it asks the store for, or from `pki/ca/pem` where it fronts no swarm name and needs a store identity | -| authelia session, JWT and storage-encryption keys | authelia's first-boot unit, in-container | `/var/lib/authelia-swarm/{session,jwt,storage-encryption}.key` | generated in place; nothing outside that container reads them | -| authelia OIDC HMAC key | same unit | `/var/lib/authelia-swarm/oidc-hmac.key` | same | -| authelia OIDC issuer key (RSA) | same unit | `/var/lib/authelia-swarm/oidc-issuer.key` | same — relying parties verify against the **public** half at `/jwks.json` | -| OIDC client secret, plaintext half | `authelia crypto hash generate --random` | `/var/lib/authelia-swarm/oidc-clients/.secret` | operator provides the file and names it in whichever option reads it — `sso.clientSecretFile` for a service; the forge and, by default, the hive collector need none — `swarm-bao-forge-oidc` and `swarm-bao-hive-otel-oidc` fetch it from the store (below) | -| OIDC client secret, digest half | the same mint | `oidc-clients/.digest` | authelia's own half; merged at runtime via `settingsFiles` | -| the swarm collector's copy of its OIDC secret | `swarm-bao-otel-oidc.service` reads it out of the swarm secret store, **on every host that runs the collector and holds a store identity** | `/var/lib/swarm-otel-oidc/.secret` inside the `swarm-otel` container | same unit, same path — one route, co-located or not. A collector with no store identity has `clientSecretFile == null`, its already-supported unauthenticated-push degrade — see below | -| Grafana's copy of its OIDC secret | `swarm-bao-grafana-oidc.service` reads it out of the swarm secret store, **on every host that runs Grafana** | `/var/lib/grafana-oidc/.secret` inside the `swarm-grafana` container | same unit, same path — one route, co-located or not. Nothing for an operator to place beyond this host's store leaf, see below | -| Grafana's admin password | `swarm-bao-grafana-admin.service` creates it in the swarm secret store at `swarm/services//admin/password` when the store holds none, then copies it out, **on every host that runs Grafana**. Grafana's leaf may create that leaf but not update it, so the store keeps the first value, and the secret publisher can't write it. To rotate, run `bao kv metadata delete secret/swarm/services//admin/password` with a token that may, such as the root token (Grafana's, the publisher's and the controller's can't), then `systemctl restart swarm-bao-grafana-admin`; a plain `bao kv delete` isn't enough, see [the callout xkey's section](#the-queues-callout-xkey) | `/var/lib/grafana-admin/password` inside the `swarm-grafana` container: `security.admin_password` for a new database, and `grafana cli admin reset-admin-password` in `grafana.service`'s `preStart` for an existing one, on every start | same unit, same path. Grafana doesn't start until the file exists, so it stays down rather than falling back to the upstream default `admin` | -| the swarm controller's copy of its OIDC secret | `swarm-controller` reads it out of the swarm secret store at start, under its own leaf | the daemon's memory only; never on disk | same read, co-located or not. Nothing for an operator to place beyond the controller's store leaf (`deploy.swarm-controller.baoClientCertFile`) | -| a hive's copy of its own OIDC secret | hive-c0re reads it out of the swarm secret store when it first connects to the queue, under the hive's own leaf | the daemon's memory only; never on disk | same read, co-located or not. Nothing for an operator to place beyond the hive's store leaf (`deploy.bao.clientCertFile`) | -| the hive collector's copy of its hive's OIDC secret | `swarm-bao-hive-otel-oidc.service` reads it out of the swarm secret store under the hive's own leaf, **on every hive that runs the collector and holds that leaf** | `/var/lib/hive-otel-oidc/hive-.secret` on the hive's host, 0400 root, loaded by `LoadCredential` | same unit, same path, co-located or not. Nothing for an operator to place beyond the hive's store leaf (`deploy.bao.clientCertFile`) | -| the swarm controller's forge token | the forge's `forgejo-swarm-controller-account`, in-container, once | `/var/lib/forgejo/swarm-controller-token` inside the forge container, the minter's record that it has minted; `swarm-bao-forge-publisher` copies it into the swarm secret store at `swarm/controller/swarm-controller/forge/token` each time the forge container starts and each time the mint writes the file; the controller holds it in memory only | nothing to place beyond the controller's store leaf. A forge host without the store needs the publisher's leaf placed (`deploy.bao.forgePublisherClientCertFile`) | -| the queue responder's copy of its OIDC secret | the responder reads it out of the swarm secret store itself as it starts, under its own leaf, and again when authelia refuses it | the daemon's memory only; never on disk | same read, wherever authelia runs. Nothing for an operator to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | -| authelia subject store | `swarmctl` and `swarm-authelia-bridge` | `users.yml` — one file, read and written by both | `swarmctl`, on the host that runs authelia | -| wireguard private key | **the operator** — `wg genkey` | whatever `deploy.wireguard.privateKeyFile` names | always operator-provided; nothing generates this for you | -| queue auth-callout nkeys (user seed + account seed) | `swarm-nats-auth ensure-callout-keys`, run by `swarm-bao-nats-callout-xkey` on the queue's host when the store holds no seed; the verb stores a seed it finds at `/var/lib/swarm-nats-callout/{callout-user,issuer}.seed` as is, then removes the file | the swarm secret store: the seeds at `swarm/services/swarm-nats/callout/{user,issuer}-seed`, read by the `swarm-nats-auth` role alone; the public halves at `swarm/public/swarm-nats/callout-{user,issuer}` | nothing to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | -| queue auth-callout xkey (curve seed) | `swarm-nats-auth ensure-callout-keys`, run by `swarm-bao-nats-callout-xkey` on the queue's host when the store holds no seed | the swarm secret store: the seed at `swarm/services/swarm-nats/callout/xkey-seed`, read by the `swarm-nats-auth` role alone; the public half at `swarm/public/swarm-nats/callout-xkey` | nothing to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | -| the queue's TLS key, copied for the gateway's `/metrics` vhost | `swarm-nats-gateway-cert.service` on the queue's host, where the gateway uses self-signed TLS: a copy of the queue's bao-PKI TLS leaf (`/var/lib/swarm-nats-tls/`) once `swarm-bao-nats-tls` has issued it, else a throwaway 30-day self-signed pair for the queue's name until then | `/var/lib/swarm-nats-gateway-tls/{cert,key}.pem`; the key is `0640` group `nginx`, read by nginx for the queue's `/metrics` vhost. The queue's own key stays `0600` root | the copy is redone on each leaf renewal (`swarm-bao-nats-tls` starts the unit, which reloads nginx if it's active); a restart before any leaf exists keeps the pair already there. Until `swarm-bao-nats-tls` issues the leaf the vhost serves the throwaway cert, so the collector's scrape fails TLS. Nothing for an operator to place | -| the secret store's own contents | openbao, on first `bao operator init` — **an operator action, not a unit** | inside the `swarm-bao` container, at its own `/var/lib/openbao`, kept across rebuilds by `ephemeral = false`. ⚠️ **Not a host path**: `nixos-container destroy swarm-bao` takes the raft data with it, so back up the container's tree, not `/var/lib/`. Only the store's TLS material (`/var/lib/swarm-bao-tls`) and its PKCS11 token (`/var/lib/swarm-bao-token`) are host-level | n/a — there is one store; a hive elsewhere is a _client_ of it and holds none of this | -| the secret store's unseal material | the HSM/TPM under `deploy.bao.seal = "pkcs11"`; openbao itself under `"shamir"` | in the token; or held by whoever ran `bao operator init`, which is what `"shamir"` means and why it's stated rather than inferred | n/a — only the host running the store seals anything | +| secret | generated by | lives at | hive elsewhere | +| -------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| swarm-services root CA (cert + key) | `swarm-bao.nix`'s granting unit, inside the store's `pki` mount | the **key** never leaves the mount; the **cert** lands at `/var/lib/swarm-bao-services-pki/services-root.pem` (0644) and at `/var/lib/hive-tls/swarm-services-root.pem` on every hive | nothing to copy — every hive receives the cert with the leaf it asks the store for, or from `pki/ca/pem` where it fronts no swarm name and needs a store identity | +| authelia session, JWT and storage-encryption keys | authelia's first-boot unit, in-container | `/var/lib/authelia-swarm/{session,jwt,storage-encryption}.key` | generated in place; nothing outside that container reads them | +| authelia OIDC HMAC key | same unit | `/var/lib/authelia-swarm/oidc-hmac.key` | same | +| authelia OIDC issuer key (RSA) | same unit | `/var/lib/authelia-swarm/oidc-issuer.key` | same — relying parties verify against the **public** half at `/jwks.json` | +| OIDC client secret, plaintext half | `authelia crypto hash generate --random` | `/var/lib/authelia-swarm/oidc-clients/.secret` | operator provides the file and names it in whichever option reads it — `sso.clientSecretFile` for a service; the forge and, by default, the hive collector need none — `swarm-bao-forge-oidc` and `swarm-bao-hive-otel-oidc` fetch it from the store (below) | +| OIDC client secret, digest half | the same mint | `oidc-clients/.digest` | authelia's own half; merged at runtime via `settingsFiles` | +| the swarm collector's copy of its OIDC secret | `swarm-bao-otel-oidc.service` reads it out of the swarm secret store, **on every host that runs the collector and holds a store identity** | `/var/lib/swarm-otel-oidc/.secret` inside the `swarm-otel` container | same unit, same path — one route, co-located or not. A collector with no store identity has `clientSecretFile == null`, its already-supported unauthenticated-push degrade — see below | +| Grafana's copy of its OIDC secret | `swarm-bao-grafana-oidc.service` reads it out of the swarm secret store, **on every host that runs Grafana** | `/var/lib/grafana-oidc/.secret` inside the `swarm-grafana` container | same unit, same path — one route, co-located or not. Nothing for an operator to place beyond this host's store leaf, see below | +| Grafana's admin password | `swarm-bao-grafana-admin.service` creates it in the swarm secret store at `swarm/services//admin/password` when the store holds none, then copies it out, **on every host that runs Grafana**. Grafana's leaf may create that leaf but not update it, so the store keeps the first value, and the secret publisher can't write it. To rotate, run `bao kv metadata delete secret/swarm/services//admin/password` with a token that may, such as the root token (Grafana's, the publisher's and the controller's can't), then `systemctl restart swarm-bao-grafana-admin`; a plain `bao kv delete` isn't enough, see [the callout xkey's section](#the-queues-callout-xkey) | `/var/lib/grafana-admin/password` inside the `swarm-grafana` container: `security.admin_password` for a new database, and `grafana cli admin reset-admin-password` in `grafana.service`'s `preStart` for an existing one, on every start | same unit, same path. Grafana doesn't start until the file exists, so it stays down rather than falling back to the upstream default `admin` | +| Grafana's metrics bearer | `swarm-bao-grafana-metrics-bearer.service` creates it in the swarm secret store at `swarm/services//metrics/bearer` when the store holds none, then copies it out, **on every host that runs Grafana**. Same grant as the admin password, create without update. To rotate, run `bao kv metadata delete secret/swarm/services//metrics/bearer` with a token that may, then `systemctl restart swarm-bao-grafana-metrics-bearer` | `/var/lib/grafana-metrics/bearer` inside the `swarm-grafana` container, holding `Bearer `: the metrics datasource's `Authorization` header. `swarm-metrics-auth` on the metrics store's host reads the stored value itself, under its own leaf | same unit, same path. The unit restarts Grafana when the file changes | +| the metrics store responder's own store leaf (cert + key) | `signLeaf` in `glue-bao-tls.nix` (`swarm-bao-pki.service`), on the host that runs the secret store, once, subject `deploy.bao.metricsAuthCommonName`. `swarm-bao-metrics-auth-policy.service` writes the `swarm-metrics-auth` role and its read grant on the bearer leaf, nothing else | `/var/lib/swarm-bao-pki/metrics-auth.pem` and `metrics-auth-key.pem` on the store's host, handed to `swarm-metrics-auth` through `LoadCredential` (paths in argv, never values) | **the operator** places a leaf with that subject on the metrics store's host and names it in `deploy.victoriametrics.authBaoClientCertFile` / `authBaoClientKeyFile`; evaluation fails naming both until then. If the store is on another host, deploy that host first: the role must exist before the responder's login succeeds, and until it does the gateway refuses every Grafana query (fail closed) | +| the swarm controller's copy of its OIDC secret | `swarm-controller` reads it out of the swarm secret store at start, under its own leaf | the daemon's memory only; never on disk | same read, co-located or not. Nothing for an operator to place beyond the controller's store leaf (`deploy.swarm-controller.baoClientCertFile`) | +| a hive's copy of its own OIDC secret | hive-c0re reads it out of the swarm secret store when it first connects to the queue, under the hive's own leaf | the daemon's memory only; never on disk | same read, co-located or not. Nothing for an operator to place beyond the hive's store leaf (`deploy.bao.clientCertFile`) | +| the hive collector's copy of its hive's OIDC secret | `swarm-bao-hive-otel-oidc.service` reads it out of the swarm secret store under the hive's own leaf, **on every hive that runs the collector and holds that leaf** | `/var/lib/hive-otel-oidc/hive-.secret` on the hive's host, 0400 root, loaded by `LoadCredential` | same unit, same path, co-located or not. Nothing for an operator to place beyond the hive's store leaf (`deploy.bao.clientCertFile`) | +| the swarm controller's forge token | the forge's `forgejo-swarm-controller-account`, in-container, once | `/var/lib/forgejo/swarm-controller-token` inside the forge container, the minter's record that it has minted; `swarm-bao-forge-publisher` copies it into the swarm secret store at `swarm/controller/swarm-controller/forge/token` each time the forge container starts and each time the mint writes the file; the controller holds it in memory only | nothing to place beyond the controller's store leaf. A forge host without the store needs the publisher's leaf placed (`deploy.bao.forgePublisherClientCertFile`) | +| the queue responder's copy of its OIDC secret | the responder reads it out of the swarm secret store itself as it starts, under its own leaf, and again when authelia refuses it | the daemon's memory only; never on disk | same read, wherever authelia runs. Nothing for an operator to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | +| authelia subject store | `swarmctl` and `swarm-authelia-bridge` | `users.yml` — one file, read and written by both | `swarmctl`, on the host that runs authelia | +| wireguard private key | **the operator** — `wg genkey` | whatever `deploy.wireguard.privateKeyFile` names | always operator-provided; nothing generates this for you | +| queue auth-callout nkeys (user seed + account seed) | `swarm-nats-auth ensure-callout-keys`, run by `swarm-bao-nats-callout-xkey` on the queue's host when the store holds no seed; the verb stores a seed it finds at `/var/lib/swarm-nats-callout/{callout-user,issuer}.seed` as is, then removes the file | the swarm secret store: the seeds at `swarm/services/swarm-nats/callout/{user,issuer}-seed`, read by the `swarm-nats-auth` role alone; the public halves at `swarm/public/swarm-nats/callout-{user,issuer}` | nothing to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | +| queue auth-callout xkey (curve seed) | `swarm-nats-auth ensure-callout-keys`, run by `swarm-bao-nats-callout-xkey` on the queue's host when the store holds no seed | the swarm secret store: the seed at `swarm/services/swarm-nats/callout/xkey-seed`, read by the `swarm-nats-auth` role alone; the public half at `swarm/public/swarm-nats/callout-xkey` | nothing to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | +| the queue's TLS key, copied for the gateway's `/metrics` vhost | `swarm-nats-gateway-cert.service` on the queue's host, where the gateway uses self-signed TLS: a copy of the queue's bao-PKI TLS leaf (`/var/lib/swarm-nats-tls/`) once `swarm-bao-nats-tls` has issued it, else a throwaway 30-day self-signed pair for the queue's name until then | `/var/lib/swarm-nats-gateway-tls/{cert,key}.pem`; the key is `0640` group `nginx`, read by nginx for the queue's `/metrics` vhost. The queue's own key stays `0600` root | the copy is redone on each leaf renewal (`swarm-bao-nats-tls` starts the unit, which reloads nginx if it's active); a restart before any leaf exists keeps the pair already there. Until `swarm-bao-nats-tls` issues the leaf the vhost serves the throwaway cert, so the collector's scrape fails TLS. Nothing for an operator to place | +| the secret store's own contents | openbao, on first `bao operator init` — **an operator action, not a unit** | inside the `swarm-bao` container, at its own `/var/lib/openbao`, kept across rebuilds by `ephemeral = false`. ⚠️ **Not a host path**: `nixos-container destroy swarm-bao` takes the raft data with it, so back up the container's tree, not `/var/lib/`. Only the store's TLS material (`/var/lib/swarm-bao-tls`) and its PKCS11 token (`/var/lib/swarm-bao-token`) are host-level | n/a — there is one store; a hive elsewhere is a _client_ of it and holds none of this | +| the secret store's unseal material | the HSM/TPM under `deploy.bao.seal = "pkcs11"`; openbao itself under `"shamir"` | in the token; or held by whoever ran `bao operator init`, which is what `"shamir"` means and why it's stated rather than inferred | n/a — only the host running the store seals anything | Authelia mints the three keys for itself, in-container, precisely because nothing outside that container ever reads them. **That's the test worth diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 54a4bc2b..6757ec9e 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -171,14 +171,29 @@ Grafana start — see [`secrets.md`](secrets.md). Neither container is reachable except through the gateway: each listens on a unix socket only the gateway's nginx can open, and VictoriaMetrics runs in a network namespace of its own with no interface but loopback. -VictoriaMetrics authenticates nothing itself. The -`metrics.` vhost forwards only the store's read API (the -Prometheus query, series, label and metadata routes, plus the vmui web UI) -without a credential. It forwards the collector's OTLP push, and the -collector's scrape of the store's own `/metrics`, only for the swarm -collector's authelia client: authelia refuses any other session or client -there. It refuses every other path, the store's write and admin -API included, with 403. The bare domain redirects to vmui. +VictoriaMetrics authenticates nothing itself; the `metrics.` +vhost does it: + +- **The read API** (the Prometheus query, series, label and metadata + routes): Grafana's datasource or an operator. Grafana sends a bearer that + `swarm-bao-grafana-metrics-bearer` creates in the swarm secret store, and + `swarm-metrics-auth` on the store's host checks it against the same store + path. A request without a bearer needs an operator's authelia session + (group `admins`); without one it gets a 401, not a login redirect. +- **vmui**: an operator's authelia session, with a redirect to the login. + The bare domain redirects to vmui. +- **The collector's OTLP push, and its scrape of the store's own + `/metrics`**: only the swarm collector's authelia client. Authelia refuses + any other session or client there, operators included. +- **Every other path**, the store's write and admin API included: 403. + +`swarm-metrics-auth` reads the store with a leaf of its own, +`deploy.victoriametrics.authBaoClientCertFile` / `authBaoClientKeyFile`. +Where the secret store runs on the same host, these default to the leaf it +mints. On any other host you place a leaf whose CN is +`deploy.bao.metricsAuthCommonName`, and evaluation refuses the config until +you set both. It reads the stored bearer at most once a minute, so a +replaced bearer starts working up to a minute later. The collector scrapes Grafana's own metrics at `https:///metrics`, so a collector on a host without diff --git a/flake.nix b/flake.nix index c1e02ea9..b63c8148 100644 --- a/flake.nix +++ b/flake.nix @@ -157,6 +157,9 @@ services.hyperhive.deploy.victorialogs.authPackage = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.swarm-logs-auth; + services.hyperhive.deploy.victoriametrics.authPackage = + lib.mkDefault + self.packages.${pkgs.stdenv.hostPlatform.system}.swarm-metrics-auth; services.hyperhive.deploy.nats.authPackage = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.swarm-nats-auth; diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index ba169276..14cb6129 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -38,6 +38,7 @@ ./glue-hive-otel-authelia.nix ./glue-hive-tls-bao-identity.nix ./glue-logs-auth-bao-identity.nix + ./glue-metrics-auth-bao-identity.nix ./glue-matrix-ctl-bao-identity.nix ./glue-matrix-oidc-client.nix ./glue-nats-auth-bao-identity.nix diff --git a/nix/host-modules/glue-bao-readers-policy-order.nix b/nix/host-modules/glue-bao-readers-policy-order.nix index 2657e961..90412d41 100644 --- a/nix/host-modules/glue-bao-readers-policy-order.nix +++ b/nix/host-modules/glue-bao-readers-policy-order.nix @@ -67,11 +67,15 @@ let ]; }; # ./swarm-grafana.nix, under Grafana's leaf, which may create the admin - # password only once that policy is written. + # password and the metrics bearer only once that policy is written. swarm-bao-grafana-admin = { here = deployCfg.grafana.enable; policies = [ "swarm-bao-grafana-oidc-policy" ]; }; + swarm-bao-grafana-metrics-bearer = { + here = deployCfg.grafana.enable; + policies = [ "swarm-bao-grafana-oidc-policy" ]; + }; # ./hive-matrix.nix, under matrix-ctl's leaf. swarm-bao-matrix-oidc = { here = deployCfg.matrix.enable; diff --git a/nix/host-modules/glue-bao-readers-store-order.nix b/nix/host-modules/glue-bao-readers-store-order.nix index 6a4776e6..0e74d121 100644 --- a/nix/host-modules/glue-bao-readers-store-order.nix +++ b/nix/host-modules/glue-bao-readers-store-order.nix @@ -46,6 +46,7 @@ let # ./swarm-grafana.nix swarm-bao-grafana-oidc = deployCfg.grafana.enable; swarm-bao-grafana-admin = deployCfg.grafana.enable; + swarm-bao-grafana-metrics-bearer = deployCfg.grafana.enable; # ./swarm-nats.nix swarm-bao-nats-callout-xkey = deployCfg.nats.enable; # ./swarm-ci-runner.nix diff --git a/nix/host-modules/glue-bao-tls.nix b/nix/host-modules/glue-bao-tls.nix index e19006d9..482609ce 100644 --- a/nix/host-modules/glue-bao-tls.nix +++ b/nix/host-modules/glue-bao-tls.nix @@ -276,6 +276,13 @@ in [ -s ${pkiDir}/logs-auth.pem ] || ${signLeaf} ${pkiDir} logs-auth \ ${lib.escapeShellArg deployCfg.bao.logsAuthCommonName} "" clientAuth + # The metrics store's Grafana auth responder, which reads Grafana's + # metrics bearer. Minted here for the matrix-ctl leaf's reason: the + # metrics store is a swarm singleton, so elsewhere this is the file an + # operator copies. + [ -s ${pkiDir}/metrics-auth.pem ] || ${signLeaf} ${pkiDir} metrics-auth \ + ${lib.escapeShellArg deployCfg.bao.metricsAuthCommonName} "" clientAuth + # The CI runner's token fetch. Minted here for the matrix-ctl leaf's # reason: the runner is a swarm singleton, so on a runner host without # the store this is the file an operator copies. diff --git a/nix/host-modules/glue-metrics-auth-bao-identity.nix b/nix/host-modules/glue-metrics-auth-bao-identity.nix new file mode 100644 index 00000000..d9efcd75 --- /dev/null +++ b/nix/host-modules/glue-metrics-auth-bao-identity.nix @@ -0,0 +1,32 @@ +# Glue: point the metrics store's Grafana auth responder at the bao leaf minted +# for it. +# +# ONE PAIRING PER FILE — the swarm-metrics-auth principal ← bao, and nothing +# else. Deleting this leaves a metrics store with no responder identity unless +# the operator names one, which ./swarm-victoriametrics.nix refuses at eval. +# +# ⚠️ The minting is NOT here. ./glue-bao-tls.nix holds the CA and signs the +# leaf. What belongs here is the pairing: which paths the responder presents. +# +# ⚠️ Gated on the store running on this host, the only place the leaf is +# minted. A metrics-store host without the store gets no default here; the +# operator places the responder's leaf and names it. +# +# Everything is `mkDefault`. An operator naming their own paths wins. +{ + lib, + config, + ... +}: +let + deployCfg = config.services.hyperhive.deploy; + pkiDir = import ./lib/bao-pki-dir.nix; +in +{ + config = lib.mkIf (deployCfg.victoriametrics.enable && deployCfg.bao.enable) { + services.hyperhive.deploy.victoriametrics = { + authBaoClientCertFile = lib.mkDefault "${pkiDir}/metrics-auth.pem"; + authBaoClientKeyFile = lib.mkDefault "${pkiDir}/metrics-auth-key.pem"; + }; + }; +} diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index 082aa2fb..fc4698ea 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -97,14 +97,43 @@ let ) ++ [ (target // { policy = "deny"; }) ]; - # The metrics store's name, every path. Only its push route and `/metrics` - # consult authelia (its read API is served with no check), and the - # collector is the one caller of both, so the rules take no `resources` - # and leave no path or query form to fall through to `default_policy`. - # ⚠️ An authenticated location added to that vhost for anyone else is - # refused here until it gets a rule ahead of the `deny`. + # The metrics store's name, every path. Authelia is consulted there for the + # collector's push route and `/metrics`, and for an operator's session on + # the read API and vmui; Grafana's bearer on the read API is checked by + # `swarm-metrics-auth`, never here. + # + # The collector's client is admitted anywhere on the name. Every other + # subject is refused the collector's two paths, then operators are admitted + # to the rest, then everything else is refused. Prefixes rather than exact + # paths for the collector's two, so no spelling of them reaches the + # operator rule. No rule takes a narrower `resources` than that, so no path + # or query form falls through to `default_policy`. metricsStoreDomain = config.services.hyperhive.swarm.victoriametrics.domain; - metricsStoreRules = collectorOnly { domain = metricsStoreDomain; }; + metricsStoreRules = + lib.optional collectorRegistered { + domain = metricsStoreDomain; + policy = "one_factor"; + subject = [ "oauth2:client:${collectorClientId}" ]; + } + ++ [ + { + domain = metricsStoreDomain; + policy = "deny"; + resources = [ + "^/opentelemetry/.*" + "^/metrics.*" + ]; + } + { + domain = metricsStoreDomain; + policy = "one_factor"; + subject = [ "group:${operatorGroup}" ]; + } + { + domain = metricsStoreDomain; + policy = "deny"; + } + ]; # One pair per `swarm.otel.publishedScrapeTargets` entry, read off the same # URL the collector requests and its token is minted for, so a target diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 5b6988cd..0cd4145d 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -908,15 +908,17 @@ let # the objects they write. # # Grafana's leaf also creates its admin password (`swarm-bao-grafana-admin` - # in ./swarm-grafana.nix). `create` without `update`, so a stored password is - # never replaced. + # in ./swarm-grafana.nix) and the bearer its metrics datasource presents + # (`swarm-bao-grafana-metrics-bearer`). `create` without `update`, so a stored + # value is never replaced. grafanaOidcReaders = [ { name = "swarm-grafana-oidc"; cn = baoDeploy.grafanaOidcCommonName; policyText = readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/oidc/client" - + createStanza "${credentialMountPath}/data/${grafanaAdminLeaf}"; + + createStanza "${credentialMountPath}/data/${grafanaAdminLeaf}" + + createStanza "${credentialMountPath}/data/${grafanaMetricsBearerLeaf}"; } ]; @@ -982,6 +984,12 @@ let # of `adminPasswordStorePath` in ./swarm-grafana.nix. grafanaAdminLeaf = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/admin/password"; + # The bearer Grafana's metrics datasource presents at the metrics store. + # Grafana's leaf creates it, the store's responder reads it. The nix half of + # `metricsBearerStorePath` in ./swarm-grafana.nix and of `--store-path` in + # ./swarm-victoriametrics.nix. + grafanaMetricsBearerLeaf = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer"; + createStanza = path: '' path "${path}" { capabilities = ["create", "read"] @@ -999,6 +1007,16 @@ let } ]; + # The metrics store's responder, which checks the bearer Grafana presents + # against the one stored for it, and reads nothing else. + metricsAuthReaders = [ + { + name = "swarm-metrics-auth"; + cn = baoDeploy.metricsAuthCommonName; + policyText = readStanza "${credentialMountPath}/data/${grafanaMetricsBearerLeaf}"; + } + ]; + # The CI runner's token fetch. The one reader of the path, which the # controller writes and cannot read back. ciRunnerReaders = [ @@ -1859,6 +1877,19 @@ in ''; }; + metricsAuthCommonName = lib.mkOption { + type = lib.types.str; + default = "swarm-metrics-auth"; + description = '' + Subject the store's `swarm-metrics-auth` cert-auth role accepts: the + identity the metrics store's Grafana auth responder presents to read + the bearer Grafana's datasource sends, + `swarm/services//metrics/bearer`, and nothing else. + + ⚠️ Reserved as a hive name by ./swarm.nix, like its siblings. + ''; + }; + ciRunnerCommonName = lib.mkOption { type = lib.types.str; default = "swarm-ci-runner"; @@ -3274,6 +3305,8 @@ in systemd.services.swarm-bao-logs-auth-policy = readerPolicyUnit "write the log store responder's agent-credential-reader bao policy and cert-auth role" logsAuthReaders; + systemd.services.swarm-bao-metrics-auth-policy = readerPolicyUnit "write the metrics store responder's Grafana-bearer-reader bao policy and cert-auth role" metricsAuthReaders; + systemd.services.swarm-bao-ci-runner-policy = readerPolicyUnit "write the CI runner's registration-token-reader bao policy and cert-auth role" ciRunnerReaders; systemd.services.swarm-bao-review-classifier-policy = readerPolicyUnit "write the review classifier's API-key-reader bao policy and cert-auth role" reviewClassifierReaders; @@ -3408,6 +3441,7 @@ in "swarm-bao-hive-leaf-policy.service" "swarm-bao-ci-runner-policy.service" "swarm-bao-logs-auth-policy.service" + "swarm-bao-metrics-auth-policy.service" "swarm-bao-forge-publisher-policy.service" ] ++ granterAfter; diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index a8e30bf0..84d10961 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -263,6 +263,16 @@ let adminPasswordPath = "/var/lib/grafana-admin/password"; hostAdminPasswordPath = "/var/lib/nixos-containers/${cfg.machine}${adminPasswordPath}"; + # The bearer the metrics datasource presents at the metrics store's query + # API, which `swarm-metrics-auth` on the store's host checks against the + # same store path. Minted once by `swarm-bao-grafana-metrics-bearer` below, + # under the same grant as the admin password. The file holds the whole + # header value, `Bearer `: nixpkgs accepts a `secureJsonData` entry + # only when it is exactly one `$__file{}`. + metricsBearerStorePath = "secret/swarm/services/${cfg.oidc.clientId}/metrics/bearer"; + metricsBearerPath = "/var/lib/grafana-metrics/bearer"; + hostMetricsBearerPath = "/var/lib/nixos-containers/${cfg.machine}${metricsBearerPath}"; + socketPath = "${deployCfg.grafana.socketDir}/grafana.sock"; # Both static NixOS ids, and both checked rather than assumed: nginx has @@ -328,9 +338,12 @@ in description = '' Where the provisioned datasource points. Defaults to the metrics store's swarm name, served by the gateway of whichever host runs the - store, so Grafana and the store need not share a host. The store's - query routes on that vhost take no credential, so the datasource - sends none. + store, so Grafana and the store need not share a host. + + The datasource sends the bearer `swarm-bao-grafana-metrics-bearer` + keeps in the swarm secret store, which is what the store's query + routes on that vhost check. Whatever URL this names receives that + bearer. ''; }; @@ -756,6 +769,86 @@ in ''; }; + # The metrics datasource's bearer: created in the store if it holds none, + # then copied into the container, under Grafana's own leaf. Same shape and + # grant as the admin password above. The store's `swarm-metrics-auth` + # reads the same path, so nothing is placed by hand on either host. + systemd.services.swarm-bao-grafana-metrics-bearer = { + description = "create Grafana's metrics store bearer in the swarm secret store if absent and fetch it"; + # Ordered after the store where it runs here: + # ./glue-bao-readers-store-order.nix. + before = [ "container@${cfg.machine}.service" ]; + wantedBy = [ + "multi-user.target" + "container@${cfg.machine}.service" + ]; + path = [ + baoDeploy.package + pkgs.coreutils + pkgs.openssl + pkgs.systemd + ]; + inherit (storeRetry) startLimitBurst startLimitIntervalSec; + serviceConfig = storeRetry.serviceConfig // { + Type = "oneshot"; + RemainAfterExit = true; + SyslogIdentifier = "swarm-bao-grafana-metrics-bearer"; + TimeoutStartSec = 30; + }; + environment = { + BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; + BAO_CLIENT_CERT = baoDeploy.grafanaOidcClientCertFile; + BAO_CLIENT_KEY = baoDeploy.grafanaOidcClientKeyFile; + } + // lib.optionalAttrs (baoDeploy.serverCaFile != null) { + BAO_CACERT = baoDeploy.serverCaFile; + }; + script = '' + set -euo pipefail + + ${atomicWriteSecret} + ${refreshConsumer} + + err="$(mktemp)" + trap 'rm -f "$err"' EXIT + + if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then + echo "could not log in to swarm-bao with Grafana's certificate; leaving Grafana's metrics bearer file as it is." >&2 + cat "$err" >&2 + exit 1 + fi + export BAO_TOKEN + + # Read, create on a miss, read again: the admin password's sequence + # above, for its reasons. + if ! bearer="$(bao kv get -field=value ${lib.escapeShellArg metricsBearerStorePath} 2>"$err")"; then + minted="$(openssl rand -hex 32)" + if ! printf '%s' "$minted" | bao kv put -cas=0 ${lib.escapeShellArg metricsBearerStorePath} value=- >/dev/null 2>"$err"; then + echo "could not create ${metricsBearerStorePath}; reading it back in case another run created it." >&2 + cat "$err" >&2 + fi + if ! bearer="$(bao kv get -field=value ${lib.escapeShellArg metricsBearerStorePath} 2>"$err")"; then + echo "swarm-bao did not return ${metricsBearerStorePath}; Grafana does not start without it." >&2 + cat "$err" >&2 + exit 1 + fi + fi + if [ -z "$bearer" ]; then + echo "swarm-bao returned an empty ${metricsBearerStorePath}; leaving the file as it is." >&2 + exit 1 + fi + + # Owner and mode as for the OIDC secret above, for its reasons. + install -d -m 0755 ${lib.escapeShellArg (builtins.dirOf hostMetricsBearerPath)} + if secret_differs ${lib.escapeShellArg hostMetricsBearerPath} "Bearer $bearer"; then + atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg hostMetricsBearerPath} "Bearer $bearer" + fi + + # `$__file{}` in a provisioned datasource is read as Grafana starts. + refresh_consumer ${lib.escapeShellArg cfg.machine} grafana.service ${lib.escapeShellArg hostMetricsBearerPath} + ''; + }; + containers.${cfg.machine} = { autoStart = true; ephemeral = false; @@ -1017,6 +1110,10 @@ in url = deployCfg.grafana.datasourceUrl; access = "proxy"; isDefault = true; + # The store's query routes admit this bearer or an operator's + # session, and nothing else. + jsonData.httpHeaderName1 = "Authorization"; + secureJsonData.httpHeaderValue1 = "$__file{${metricsBearerPath}}"; } { name = "VictoriaLogs"; diff --git a/nix/host-modules/swarm-victoriametrics.nix b/nix/host-modules/swarm-victoriametrics.nix index f1f0d969..50bf341f 100644 --- a/nix/host-modules/swarm-victoriametrics.nix +++ b/nix/host-modules/swarm-victoriametrics.nix @@ -36,13 +36,14 @@ let # container, which knows it as `gateway-nginx`. nginxGid = config.ids.gids.nginx; - # The store's read API: the only part of it the vhost serves without a - # credential. Each location's `proxy_pass` carries the path, so the store - # receives the path nginx matched, not the caller's unnormalised spelling. + # The store's read API, which the vhost forwards to a caller holding + # Grafana's bearer or an operator's session. Each location's `proxy_pass` + # carries the path, so the store receives the path nginx matched, not the + # caller's unnormalised spelling. # # Grafana's Prometheus datasource reads the query, series, label, metadata, # buildinfo, exemplar, rules and alerts routes; the store answers the last - # three with fixed placeholders. vmui is the static files under `/vmui/` + # three with fixed placeholders. vmui (`uiLocations` below) is static files # plus the query API and the remaining read-only routes here. Paths read off # the pinned build's `app/vmselect/main.go` and vmui's `src/`. readPaths = [ @@ -64,22 +65,67 @@ let "/expand-with-exprs" "/prettify-query" "/metric-relabel-debug" - "/vmui" - ]; - # The store serves only `/values` under `/api/v1/label/`, and only - # static files and UI config under `/vmui/`. - readPrefixes = [ - "/api/v1/label/" - "/vmui/" ]; + # The store serves only `/values` under `/api/v1/label/`. + readPrefixes = [ "/api/v1/label/" ]; + + # Bare `auth_request`, no login redirect: a machine caller handed + # `error_page 401 =302` follows it and reads authelia's login page as a 200 + # holding no series. `/__metrics_read_auth` picks the check from the + # caller's `Authorization` (`appendHttpConfig` below). + readAuth = '' + auth_request /__metrics_read_auth; + ''; readLocations = let - to = path: { proxyPass = upstream path; }; + to = path: { + proxyPass = upstream path; + extraConfig = readAuth; + }; in lib.listToAttrs ( map (p: lib.nameValuePair "= ${p}" (to p)) readPaths ++ map (p: lib.nameValuePair "^~ ${p}" (to p)) readPrefixes ); + + # The same three lines as `swarmAuthRequest` in ./swarm-victorialogs.nix, + # for the same browser: an operator without a session is sent to log in. + swarmAuthRequest = '' + auth_request /__metrics_authelia; + auth_request_set $target_url $scheme://$http_host$request_uri; + error_page 401 =302 https://${hyperhiveCfg.swarm.authelia.domain}/?rd=$target_url; + ''; + # vmui's static files and UI config. Only a browser loads these, and its + # queries then go to the read locations above with the same session cookie. + uiLocations = { + "= /vmui" = { + proxyPass = upstream "/vmui"; + extraConfig = swarmAuthRequest; + }; + "^~ /vmui/" = { + proxyPass = upstream "/vmui/"; + extraConfig = swarmAuthRequest; + }; + }; + + # The `auth_request` responder for Grafana's bearer (`swarm-metrics-auth`). + # Its socket's directory holds that socket alone and is traverse-only for + # others, the shape `swarm-logs-auth` in ./swarm-victorialogs.nix has. + metricsAuthUnit = "swarm-metrics-auth"; + metricsAuthSocket = "/run/${metricsAuthUnit}/auth.sock"; + metricsAuthCredential = id: "/run/credentials/${metricsAuthUnit}.service/${id}"; + # Credential id → host path of the responder's store identity. + metricsAuthStoreFiles = { + "bao-client.pem" = deployCfg.victoriametrics.authBaoClientCertFile; + "bao-client-key.pem" = deployCfg.victoriametrics.authBaoClientKeyFile; + } + // lib.optionalAttrs (deployCfg.bao.serverCaFile != null) { + "bao-ca.pem" = deployCfg.bao.serverCaFile; + }; + baoCfg = hyperhiveCfg.swarm.bao; + # The path Grafana's host creates the bearer at, under the KV mount. The + # same leaf as `grafanaMetricsBearerLeaf` in ./swarm-bao.nix. + grafanaBearerStorePath = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer"; in { # What the store IS from any hive's point of view, the name it answers on, @@ -113,6 +159,44 @@ in ''; }; + options.services.hyperhive.deploy.victoriametrics.authPackage = lib.mkOption { + type = lib.types.package; + defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-metrics-auth"; + description = '' + The `swarm-metrics-auth` package: the gateway's `auth_request` responder + that checks the bearer Grafana's metrics datasource presents on the + store's query API against the secret store. + ''; + }; + + options.services.hyperhive.deploy.victoriametrics.authBaoClientCertFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-bao-pki/metrics-auth.pem"; + description = '' + Client certificate `swarm-metrics-auth` presents to the swarm's secret + store to read Grafana's metrics bearer. Its subject must be + {option}`services.hyperhive.deploy.bao.metricsAuthCommonName`. + + Required when `deploy.victoriametrics.enable` is set; unset fails at + eval. ./glue-metrics-auth-bao-identity.nix points it at the leaf + ./glue-bao-tls.nix mints on a host that runs the store; on any other + metrics-store host it is a file the operator places. + + A path, never a value. + ''; + }; + + options.services.hyperhive.deploy.victoriametrics.authBaoClientKeyFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-bao-pki/metrics-auth-key.pem"; + description = '' + Private key for {option}`services.hyperhive.deploy.victoriametrics.authBaoClientCertFile`. + A path, never a value. + ''; + }; + config = lib.mkIf deployCfg.victoriametrics.enable { assertions = [ { @@ -128,6 +212,22 @@ in cannot connect and every read, push and scrape fails with 502. ''; } + { + assertion = + deployCfg.victoriametrics.authBaoClientCertFile != null + && deployCfg.victoriametrics.authBaoClientKeyFile != null; + message = '' + services.hyperhive.deploy.victoriametrics.authBaoClientCertFile and + authBaoClientKeyFile must both be set. + + swarm-metrics-auth reads Grafana's metrics bearer from the swarm's + secret store with this certificate. They default to the leaf the + store mints where services.hyperhive.deploy.bao.enable is set on + this host; elsewhere, issue a leaf whose CN is + services.hyperhive.deploy.bao.metricsAuthCommonName and name it in + both options. + ''; + } ]; # The gateway name, inside `deployCfg.victoriametrics.enable` — that @@ -141,77 +241,118 @@ in services.hyperhive.gateway.enable = lib.mkDefault true; services.hyperhive.gateway.dns.enable = lib.mkDefault true; - services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // { - listen = gatewayCfg.lib.listen; - extraConfig = gatewayCfg.lib.securityHeaders; - locations = readLocations // { - # Nothing reaches the store unless a location here names it. The - # store serves writes and admin calls (`/api/v1/import`, - # `/api/v1/admin/tsdb/delete_series`, ...) on the same listener as - # its queries, and authenticates none of them. - "/".return = "403"; + # Which check a read location's `auth_request` gets. `auth_request` takes a + # fixed URI, so it names `/__metrics_read_auth`, which rewrites to the + # target this map picks: a bearer goes to Grafana's check, anything else + # (an operator's session cookie) to authelia. + services.nginx.appendHttpConfig = '' + map $http_authorization $swarm_metrics_read_auth_target { + "~^Bearer " /__metrics_grafana_auth; + default /__metrics_authelia; + } + ''; - # The swarm UI's quick link opens the bare domain. The store's own - # index page links mostly to paths refused above, so send it to vmui. - "= /".return = "302 /vmui/"; + # `removeAttrs`/`forceSSL`: authelia answers a plain-http auth subrequest + # with 400, which `auth_request` cannot read as anything but a broken + # check. Same as ./swarm-victorialogs.nix's vhost. + services.nginx.virtualHosts."${cfg.domain}" = + (builtins.removeAttrs (gatewayCfg.lib.tlsFor cfg.domain) [ "addSSL" ]) + // { + forceSSL = true; + listen = gatewayCfg.lib.listen; + extraConfig = gatewayCfg.lib.securityHeaders; + locations = + readLocations + // uiLocations + // { + # Nothing reaches the store unless a location here names it. The + # store serves writes and admin calls (`/api/v1/import`, + # `/api/v1/admin/tsdb/delete_series`, ...) on the same listener as + # its queries, and authenticates none of them. + "/".return = "403"; - # The swarm collector's ingest route. `=` so it outranks the `/` - # prefix above. - # - # ⚠️ No `error_page 401 =302` here, and its absence is the point: a - # redirect is right for a browser and wrong for a pusher, which would - # follow it and POST its batch at a login page that answers 200 — - # ingest reporting healthy while storing nothing. - "= /opentelemetry/api/v1/push" = { - proxyPass = upstream "/opentelemetry/api/v1/push"; - extraConfig = '' - auth_request /__metrics_push_authz; - ''; - }; + # The swarm UI's quick link opens the bare domain. The store's own + # index page links mostly to paths refused above, so send it to vmui. + "= /".return = "302 /vmui/"; - # The store's own health, for the swarm collector wherever it runs: - # served on the query listener, so no exporter and no extra port. `=` - # for the same reason as the push route, and no `error_page` for the - # same reason: a scraper would parse a login page as metrics. - # - # The collector requests exactly this URL, declared as - # `publishedScrapeTargets.victoriametrics` in ./swarm-otel-service.nix; - # the two must agree, since that string is also the token's audience. - "= /metrics" = { - proxyPass = upstream "/metrics"; - extraConfig = '' - auth_request /__metrics_push_authz; - ''; - }; + # The swarm collector's ingest route. `=` so it outranks the `/` + # prefix above. + # + # ⚠️ No `error_page 401 =302` here, and its absence is the point: a + # redirect is right for a browser and wrong for a pusher, which would + # follow it and POST its batch at a login page that answers 200 — + # ingest reporting healthy while storing nothing. + "= /opentelemetry/api/v1/push" = { + proxyPass = upstream "/opentelemetry/api/v1/push"; + extraConfig = '' + auth_request /__metrics_authelia; + ''; + }; - # The subrequest for both gated locations above. Authelia admits the - # collector's client anywhere on this name and refuses every other - # subject (`metricsStoreRules` in ./swarm-authelia.nix), so the scrape - # needs no subrequest of its own. Same target and header set as the - # sibling log store's, which took them from `swarm-ui.nix` — - # `X-Original-URL` and `X-Original-Method` are what authelia's - # auth-request implementation reads, and the address it compares the - # token's audience against. - "= /__metrics_push_authz" = { - proxyPass = "https://${hyperhiveCfg.swarm.authelia.domain}/api/authz/auth-request"; - # nixpkgs appends its OWN `Host $host` after extraConfig, which - # would override verifiedProxyTo's — see the comment on - # verifiedProxyTo in hive-gateway/vhost-lib.nix. - recommendedProxySettings = false; - extraConfig = '' - internal; - ${gatewayCfg.lib.verifiedProxyTo hyperhiveCfg.swarm.authelia.domain} - proxy_pass_request_body off; - proxy_set_header Content-Length ""; - proxy_set_header X-Original-Method $request_method; - proxy_set_header X-Original-URL $scheme://$http_host$request_uri; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Host $http_host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - ''; - }; + # The store's own health, for the swarm collector wherever it runs: + # served on the query listener, so no exporter and no extra port. `=` + # for the same reason as the push route, and no `error_page` for the + # same reason: a scraper would parse a login page as metrics. + # + # The collector requests exactly this URL, declared as + # `publishedScrapeTargets.victoriametrics` in ./swarm-otel-service.nix; + # the two must agree, since that string is also the token's audience. + "= /metrics" = { + proxyPass = upstream "/metrics"; + extraConfig = '' + auth_request /__metrics_authelia; + ''; + }; + + # `rewrite … last` re-matches locations, so the subrequest lands in + # one of the two internal locations named by the map. + "= /__metrics_read_auth" = { + extraConfig = '' + internal; + rewrite ^ $swarm_metrics_read_auth_target last; + ''; + }; + + # Grafana's bearer, checked by `swarm-metrics-auth`. The + # `Authorization` header rides the subrequest unchanged. No rate limit + # in front: the responder reads the store at most once per its cache + # TTL, whatever the bearers. + "= /__metrics_grafana_auth" = { + proxyPass = "http://unix:${metricsAuthSocket}:/"; + extraConfig = '' + internal; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + ''; + }; + + # The authelia subrequest. For the collector's two locations above it + # admits only the collector's client; for the read and vmui locations + # it admits an operator's session (`metricsStoreRules` in + # ./swarm-authelia.nix). Same target and header set as the sibling log + # store's, which took them from `swarm-ui.nix` — `X-Original-URL` and + # `X-Original-Method` are what authelia's auth-request implementation + # reads, and the address it compares the token's audience against. + "= /__metrics_authelia" = { + proxyPass = "https://${hyperhiveCfg.swarm.authelia.domain}/api/authz/auth-request"; + # nixpkgs appends its OWN `Host $host` after extraConfig, which + # would override verifiedProxyTo's — see the comment on + # verifiedProxyTo in hive-gateway/vhost-lib.nix. + recommendedProxySettings = false; + extraConfig = '' + internal; + ${gatewayCfg.lib.verifiedProxyTo hyperhiveCfg.swarm.authelia.domain} + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-Method $request_method; + proxy_set_header X-Original-URL $scheme://$http_host$request_uri; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + ''; + }; + }; }; - }; # The socket directory, created host-side before the container starts: # nixos-container refuses to start when a bind source is missing. @@ -224,6 +365,67 @@ in "d ${socketDir} 0770 root ${toString nginxGid} - -" ]; + # Grafana's check on the read API. On the host rather than in the store's + # container: it needs nothing of the store, only nginx's reach to its + # socket and its own store identity. + users.users.${metricsAuthUnit} = { + isSystemUser = true; + group = metricsAuthUnit; + description = "swarm metrics store auth_request responder"; + }; + users.groups.${metricsAuthUnit} = { }; + + systemd.services.${metricsAuthUnit} = { + description = "swarm metrics store auth_request responder for Grafana's bearer"; + wantedBy = [ "multi-user.target" ]; + # Where the store's PKI is minted on this host, the leaf is one of its + # files. Ordering only: elsewhere the unit does not exist. + after = [ + "network.target" + "swarm-bao-pki.service" + ]; + serviceConfig = { + ExecStart = lib.concatStringsSep " " [ + "${deployCfg.victoriametrics.authPackage}/bin/swarm-metrics-auth" + "--socket ${metricsAuthSocket}" + # The role ./swarm-bao.nix writes on the store's `cert` mount. + "--store-cert-role swarm-metrics-auth" + "--store-path ${grafanaBearerStorePath}" + ]; + # Paths only: `argv` is world-readable via /proc//cmdline. + LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") metricsAuthStoreFiles; + # ⚠️ A static user, never `DynamicUser`: with `RuntimeDirectoryPreserve` + # systemd puts a dynamic user's runtime directory under + # `/run/private/` (0700 root:root), which nginx cannot traverse. + User = metricsAuthUnit; + Group = metricsAuthUnit; + RuntimeDirectory = metricsAuthUnit; + # Traverse-only for others, so nginx reaches the socket path. + RuntimeDirectoryMode = "0751"; + # The path never vanishes from under a running nginx; the responder + # unlinks a stale socket as it starts. + RuntimeDirectoryPreserve = "yes"; + Restart = "on-failure"; + RestartSec = "5s"; + SyslogIdentifier = metricsAuthUnit; + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = "disconnected"; + PrivateDevices = true; + NoNewPrivileges = true; + RestrictSUIDSGID = true; + RemoveIPC = true; + }; + environment = { + BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; + BAO_CLIENT_CERT = metricsAuthCredential "bao-client.pem"; + BAO_CLIENT_KEY = metricsAuthCredential "bao-client-key.pem"; + } + // lib.optionalAttrs (metricsAuthStoreFiles ? "bao-ca.pem") { + BAO_CACERT = metricsAuthCredential "bao-ca.pem"; + }; + }; + containers.${cfg.machine} = { autoStart = true; ephemeral = false; diff --git a/nix/host-modules/swarm.nix b/nix/host-modules/swarm.nix index ee393eba..90ffbdcc 100644 --- a/nix/host-modules/swarm.nix +++ b/nix/host-modules/swarm.nix @@ -56,6 +56,7 @@ let deployCfg.bao.natsAuthCommonName deployCfg.bao.ciRunnerCommonName deployCfg.bao.logsAuthCommonName + deployCfg.bao.metricsAuthCommonName deployCfg.bao.reviewClassifierCommonName deployCfg.bao.builderCommonName deployCfg.bao.forgePublisherCommonName diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index e6574e7c..592802b3 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -169,7 +169,7 @@ let _: u: (u.environment.BAO_CLIENT_CERT or null) == granterCertFile ) baoGrantWithConsumers.systemd.services; - # The eighteen units that write a `swarm-*` grant, by name, for the discovery + # The nineteen units that write a `swarm-*` grant, by name, for the discovery # control below. grantingUnitNames = [ "swarm-bao-controller-policy" @@ -187,6 +187,7 @@ let "swarm-bao-hive-leaf-policy" "swarm-bao-ci-runner-policy" "swarm-bao-logs-auth-policy" + "swarm-bao-metrics-auth-policy" "swarm-bao-forge-publisher-policy" "swarm-bao-review-classifier-policy" "swarm-bao-builder-policy" @@ -593,10 +594,10 @@ let # both, and the two are separate principals precisely because a # dashboard is not entitled to a collector's credential. # - # The admin password is the one leaf it may create, and without `update`: - # a grant that could replace it would let a second run rotate the - # password Grafana's database already holds. - name = "the Grafana reader reads its own client secret and creates its admin password, and nothing else"; + # The admin password and the metrics bearer are the leaves it may create, + # and without `update`: a grant that could replace one would let a second + # run rotate a value Grafana or the metrics store already holds. + name = "the Grafana reader reads its own client secret and creates its admin password and metrics bearer, and nothing else"; ok = let s = baoGrantHere.systemd.services.swarm-bao-grafana-oidc-policy.script; @@ -613,6 +614,13 @@ let "read" ]; } + { + path = "secret/data/swarm/services/swarm-grafana/metrics/bearer"; + caps = [ + "create" + "read" + ]; + } ] && !(lib.hasInfix "secret/data/swarm/agents" s) && !(lib.hasInfix "secret/data/swarm/hives" s) @@ -731,6 +739,32 @@ let in lib.hasInfix "/logs-auth.pem ]" s && lib.hasInfix "swarm-logs-auth \"\" clientAuth" s; } + { + # Read-only on the one leaf Grafana's host creates, and nothing else: + # this principal checks a presented bearer and does nothing with it. + name = "the metrics store responder's grant is Grafana's metrics bearer, read-only, and nothing else"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-metrics-auth-policy.script; + in + grantsIn s == [ + { + path = "secret/data/swarm/services/swarm-grafana/metrics/bearer"; + caps = [ "read" ]; + } + ] + && lib.hasInfix "auth/cert/certs/swarm-metrics-auth" s + && lib.hasInfix "allowed_common_names=swarm-metrics-auth" s + && lib.hasInfix "token_policies=swarm-metrics-auth" s; + } + { + name = "the PKI unit signs the metrics store responder's leaf under its own subject"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-pki.script; + in + lib.hasInfix "/metrics-auth.pem ]" s && lib.hasInfix "swarm-metrics-auth \"\" clientAuth" s; + } { # 🩸 The half that makes the policies above bind: a policy grants only # through a token that carries it, and a token is minted by a cert-auth @@ -942,24 +976,24 @@ let } { # A store host without the granter's pair writes its grants some other - # way, so none of the eighteen units may exist. Without this arm + # way, so none of the nineteen units may exist. Without this arm # `lib.mkIf haveGranter` could be dropped from any of them and every other # case here would still pass. - name = "without the granter's pair none of the eighteen granting units render"; + name = "without the granter's pair none of the nineteen granting units render"; ok = let s = baoGranterOptOut.systemd.services; in lib.all (unit: !(s ? ${unit})) (grantingUnitNames ++ [ "swarm-bao-granter-role" ]) - # The control: the same store with the pair renders all eighteen. + # The control: the same store with the pair renders all nineteen. && lib.all (unit: baoGrantHere.systemd.services ? ${unit}) grantingUnitNames; } { - # 🩸 What replaced the silent skip. With no bootstrap token the eighteen still + # 🩸 What replaced the silent skip. With no bootstrap token the nineteen still # render, and a refused granter fails them with the step that fixes it. # A store host that never named a token is told to name one, since the # unit that sets the granter up renders only where it has. - name = "a store host without a bootstrap token renders the eighteen, each failing loudly with the one-time step"; + name = "a store host without a bootstrap token renders the nineteen, each failing loudly with the one-time step"; ok = let s = baoGranterNoToken.systemd.services; @@ -1049,6 +1083,7 @@ let "swarm-services-cert" "swarm-bao-forwarder-oidc" "swarm-bao-grafana-admin" + "swarm-bao-grafana-metrics-bearer" ] ++ policyReaders ); @@ -1776,8 +1811,8 @@ let } { # What makes the case above mean something: discovery by the granter's - # certificate reaches all eighteen units, and each yields calls. - name = "the granter-policy check sees all eighteen granting units, and parses calls from each"; + # certificate reaches all nineteen units, and each yields calls. + name = "the granter-policy check sees all nineteen granting units, and parses calls from each"; ok = lib.sort lib.lessThan (lib.attrNames granterUnits) == lib.sort lib.lessThan grantingUnitNames && lib.all (u: baoCalls u.script != [ ]) (lib.attrValues granterUnits) diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index 986aece1..f9fe80bb 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -478,36 +478,94 @@ let && lib.hasInfix "error_page" browser; } { - # The store authenticates nothing itself, so every location that - # forwards without `auth_request` is open to anything that can reach the - # domain. The query location is the positive control: it shows the - # filter finds forwarding locations at all. - name = "the metrics vhost forwards only read paths without a credential"; + # The store authenticates nothing itself, so a location that forwards to + # it without `auth_request` is open to anything that can reach the + # domain. The query location is the control: it shows the filter finds + # store-forwarding locations at all. + name = "no location on the metrics vhost forwards to the store without auth_request"; ok = let locs = allLocal.services.nginx.virtualHosts."metrics.t.local".locations; - open = lib.filterAttrs ( - _: l: - l.proxyPass != null - && !(lib.hasInfix "auth_request" l.extraConfig) - && !(lib.hasInfix "internal;" l.extraConfig) + toStore = lib.filterAttrs ( + _: l: l.proxyPass != null && lib.hasPrefix "http://unix:/run/swarm-victoriametrics/" l.proxyPass ) locs; - writeWords = [ - "import" - "write" - "admin" - "delete" - "insert" - "internal" - "push" - "prometheus" - "influx" - ]; + open = lib.filterAttrs (_: l: !(lib.hasInfix "auth_request " l.extraConfig)) toStore; in - locs."/".proxyPass == null - && locs."/".return == "403" - && open ? "= /api/v1/query" - && lib.all (n: n != "/" && !(lib.any (w: lib.hasInfix w n) writeWords)) (lib.attrNames open); + toStore ? "= /api/v1/query" + && open == { } + && locs."/".proxyPass == null + && locs."/".return == "403"; + } + { + # The read API's check, link by link: every read location names the + # switch with no login redirect (a machine reader would get login HTML + # as a 200), the switch rewrites to the map's pick, a bearer reaches + # the responder's socket, which the unit binds, and anything else goes + # to authelia. vmui is a browser's, so it redirects to the login. + name = "the metrics read API sends a bearer to swarm-metrics-auth and a session to authelia"; + ok = + let + l = allLocal.services.nginx.virtualHosts."metrics.t.local".locations; + http = allLocal.services.nginx.appendHttpConfig; + unit = allLocal.systemd.services.swarm-metrics-auth; + reads = lib.filterAttrs (n: _: lib.hasPrefix "= /api/v1/" n || lib.hasPrefix "^~ /api/v1/" n) l; + in + reads ? "= /api/v1/query_range" + && lib.all ( + r: + lib.hasInfix "auth_request /__metrics_read_auth;" r.extraConfig + && !(lib.hasInfix "error_page" r.extraConfig) + ) (lib.attrValues reads) + && lib.hasInfix "map \$http_authorization \$swarm_metrics_read_auth_target" http + && lib.hasInfix "\"~^Bearer \" /__metrics_grafana_auth;" http + && lib.hasInfix "default /__metrics_authelia;" http + && + lib.hasInfix "rewrite ^ \$swarm_metrics_read_auth_target last;" + l."= /__metrics_read_auth".extraConfig + && lib.hasInfix "internal;" l."= /__metrics_grafana_auth".extraConfig + && l."= /__metrics_grafana_auth".proxyPass == "http://unix:/run/swarm-metrics-auth/auth.sock:/" + && lib.hasInfix "--socket /run/swarm-metrics-auth/auth.sock" unit.serviceConfig.ExecStart + && lib.hasInfix "auth_request /__metrics_authelia;" l."^~ /vmui/".extraConfig + && lib.hasInfix "error_page 401 =302" l."^~ /vmui/".extraConfig + && l."= /__metrics_authelia".proxyPass == "https://auth.t.local/api/authz/auth-request"; + } + { + # Grafana's host creates the bearer and its datasource sends it; the + # store's responder reads it. Three modules spell the path, and the + # store's grants name it twice: if any one drifts, Grafana sends a value + # the responder never reads, or one of them is refused by the store. + name = "Grafana's datasource sends the bearer swarm-metrics-auth checks, at one store path"; + ok = + let + leaf = "swarm/services/${allLocal.services.hyperhive.swarm.grafana.oidc.clientId}/metrics/bearer"; + ds = + lib.findFirst (d: d.uid == "swarm-victoriametrics") null + allLocal.containers.swarm-grafana.config.services.grafana.provision.datasources.settings.datasources; + bearerUnit = allLocal.systemd.services.swarm-bao-grafana-metrics-bearer; + s = allLocal.systemd.services; + in + ds.jsonData.httpHeaderName1 == "Authorization" + && ds.secureJsonData.httpHeaderValue1 == "$__file{/var/lib/grafana-metrics/bearer}" + && lib.hasInfix "bao kv put -cas=0 secret/${leaf} value=-" bearerUnit.script + && lib.hasInfix "/var/lib/nixos-containers/swarm-grafana/var/lib/grafana-metrics/bearer \"Bearer $bearer\"" bearerUnit.script + && lib.hasInfix "--store-path ${leaf}" s.swarm-metrics-auth.serviceConfig.ExecStart + && lib.hasInfix "path \"secret/data/${leaf}\" {\n capabilities = [\"read\"]\n}" s.swarm-bao-metrics-auth-policy.script + && lib.hasInfix "path \"secret/data/${leaf}\" {\n capabilities = [\"create\", \"read\"]\n}" s.swarm-bao-grafana-oidc-policy.script; + } + { + # The responder reads the store as itself, with the leaf the store's + # host mints for it. + name = "swarm-metrics-auth presents its own store leaf"; + ok = + let + v = allLocal.services.hyperhive.deploy.victoriametrics; + lc = allLocal.systemd.services.swarm-metrics-auth.serviceConfig.LoadCredential; + in + lib.hasSuffix "/metrics-auth.pem" v.authBaoClientCertFile + && lib.hasSuffix "/metrics-auth-key.pem" v.authBaoClientKeyFile + && builtins.elem "bao-client.pem:${v.authBaoClientCertFile}" lc + && builtins.elem "bao-client-key.pem:${v.authBaoClientKeyFile}" lc + && lib.hasInfix "/metrics-auth.pem ]" allLocal.systemd.services.swarm-bao-pki.script; } { # The read counterpart to the ingest location: an agent queries the log @@ -592,7 +650,7 @@ let # `/run/private/` (0700 root:root), which nginx cannot traverse, so a # socket there answers every proxied request with a 502. Each `unix:` # upstream under `/run//` is matched to the units creating ``. - # The first clause is the control: the match finds swarm-logs-auth's. + # The first two clauses are the control: the match finds both responders. name = "no unix socket nginx proxies to sits in a DynamicUser's preserved runtime directory"; ok = let @@ -626,7 +684,9 @@ let "no" ]); in - owners ? swarm-logs-auth && !(lib.any (u: hidden u.serviceConfig) (lib.attrValues owners)); + owners ? swarm-logs-auth + && owners ? swarm-metrics-auth + && !(lib.any (u: hidden u.serviceConfig) (lib.attrValues owners)); } { # Every agent bearer the responder has not recently granted costs a diff --git a/nix/module-eval/swarm-otel-identity.nix b/nix/module-eval/swarm-otel-identity.nix index 2d27d63c..45e8f1ac 100644 --- a/nix/module-eval/swarm-otel-identity.nix +++ b/nix/module-eval/swarm-otel-identity.nix @@ -287,20 +287,21 @@ let in m != null && m.proxyPass == "http://unix:/run/swarm-victoriametrics/victoriametrics.sock:/metrics" - && lib.hasInfix "auth_request /__metrics_push_authz;" m.extraConfig + && lib.hasInfix "auth_request /__metrics_authelia;" m.extraConfig && !(lib.hasInfix "error_page" m.extraConfig) - && lib.hasInfix "internal;" locs."= /__metrics_push_authz".extraConfig + && lib.hasInfix "internal;" locs."= /__metrics_authelia".extraConfig && locs."/".return == "403" && locs."/".proxyPass == null; } { - # The metrics store's name admits the collector's client on every path - # and refuses every other subject. The rules carry no `resources`, so no - # path or query spelling misses them and reaches `default_policy`, and - # they come before the apex rule. That is only right while the push - # route and `/metrics` are the vhost's sole authelia-gated locations, - # which the last clause pins. - name = "authelia admits only the collector anywhere on the metrics store's name"; + # The metrics store's name admits the collector's client on every path, + # an operator on every path but the collector's two, and nobody else. + # Only the collector's two paths carry `resources`, as prefixes, so no + # path or query spelling of them reaches the operator rule, and no other + # spelling reaches `default_policy`. The rules come before the apex + # rule. The collector's two locations call authelia directly, so a + # bearer there never reaches Grafana's check. + name = "authelia admits the collector anywhere on the metrics store's name and operators only off its two paths"; ok = let h = allLocal.services.hyperhive; @@ -311,38 +312,33 @@ let storeRules = lib.filter (r: r.domain == "metrics.t.local") indexed; apex = lib.findFirst (r: r.domain == "t.local") null indexed; policy = subject: autheliaPolicy allLocal subject "metrics.t.local"; - paths = [ + collectorPaths = [ "/opentelemetry/api/v1/push" + "/opentelemetry/api/v1/push?x=1" "/metrics" "/metrics?x=1" + ]; + readPaths = [ "/api/v1/query" + "/api/v1/query_range?query=up" + "/vmui/" ]; locs = allLocal.services.nginx.virtualHosts.${h.swarm.victoriametrics.domain}.locations; in - map (r: removeAttrs r [ "i" ]) storeRules == [ - { - domain = "metrics.t.local"; - policy = "one_factor"; - subject = [ collector ]; - } - { + lib.length storeRules == 4 + && + removeAttrs (lib.last storeRules) [ "i" ] == { domain = "metrics.t.local"; policy = "deny"; } - ] && apex != null && lib.all (r: r.i < apex.i) storeRules - && lib.all (p: policy collector p == "one_factor") paths - && lib.all (p: policy "user:someone" p == "deny") paths - && lib.all (p: policy "group:admins" p == "deny") paths - && - lib.hasInfix "auth_request /__metrics_push_authz;" - locs."= /opentelemetry/api/v1/push".extraConfig - && - lib.attrNames (lib.filterAttrs (_: l: lib.hasInfix "auth_request" (l.extraConfig or "")) locs) == [ - "= /metrics" - "= /opentelemetry/api/v1/push" - ]; + && lib.all (p: policy collector p == "one_factor") (collectorPaths ++ readPaths) + && lib.all (p: policy "user:someone" p == "deny") (collectorPaths ++ readPaths) + && lib.all (p: policy "group:admins" p == "deny") collectorPaths + && lib.all (p: policy "group:admins" p == "one_factor") readPaths + && lib.hasInfix "auth_request /__metrics_authelia;" locs."= /opentelemetry/api/v1/push".extraConfig + && lib.hasInfix "auth_request /__metrics_authelia;" locs."= /metrics".extraConfig; } { # Every published target admits the collector's client and refuses any @@ -398,7 +394,9 @@ let let rules = autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules; - metricsRules = lib.filter (r: r ? resources) rules; + # The metrics store's own name is the case above's; its rules carry + # `resources` for its push route and `/metrics` together. + metricsRules = lib.filter (r: r ? resources && r.domain != "metrics.t.local") rules; re = lib.head (lib.head metricsRules).resources; table = [ { diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix index 95ae8613..b74aab88 100644 --- a/nix/module-eval/swarm-services-switch.nix +++ b/nix/module-eval/swarm-services-switch.nix @@ -540,7 +540,7 @@ let { # No interface but loopback, so the socket is the store's only listener # the host can reach. Every location that forwards to the store goes - # through it; the authelia subrequest is the one `https://` upstream. + # through it; the internal auth subrequests are the only other upstreams. # The read, push and scrape locations are named so an empty filter # can't pass. name = "the metrics store runs on a netns of its own and the gateway reaches it over its socket"; @@ -553,8 +553,10 @@ let viaSock = path: "http://unix:${sock}:${path}"; locs = host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.victoriametrics.domain}.locations; - toStore = lib.filter (p: p != null && !(lib.hasPrefix "https://" p)) ( - map (l: l.proxyPass) (lib.attrValues locs) + toStore = map (l: l.proxyPass) ( + lib.filter (l: l.proxyPass != null && !(lib.hasInfix "internal;" l.extraConfig)) ( + lib.attrValues locs + ) ); unit = c.config.systemd.services.victoriametrics; nginxGid = host.ids.gids.nginx; @@ -749,6 +751,7 @@ let "swarm-bao-hive-otel-oidc" "swarm-bao-grafana-oidc" "swarm-bao-grafana-admin" + "swarm-bao-grafana-metrics-bearer" "swarm-bao-nats-callout-xkey" "swarm-bao-ci-runner" "swarm-bao-otel-oidc" @@ -771,7 +774,7 @@ let !(lib.any (n: lib.elem n (s.after ++ s.wants ++ s.requires)) storeUnits); in lib.all waits readers - && lib.length readersElsewhere == 6 + && lib.length readersElsewhere == 7 && lib.all namesNoStoreUnit readersElsewhere && !(lib.any (u: storeOnlyNoHive.systemd.services ? ${u}) readers); } diff --git a/nix/packages/default.nix b/nix/packages/default.nix index bfcafd6b..a69602bb 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -167,6 +167,10 @@ in # `daemonBins` like the responder above: it runs on the log store's host only. swarm-logs-auth = mkBinPackage "swarm-logs-auth" "hyperhive swarm log store agent auth responder"; + # The metrics store gateway's `auth_request` responder for Grafana's bearer. Out + # of `daemonBins` like the two above: it runs on the metrics store's host only. + swarm-metrics-auth = mkBinPackage "swarm-metrics-auth" "hyperhive swarm metrics store Grafana auth responder"; + # The matrix container's control binary; `mint` publishes the sender token. Out of `daemonBins` for # "runs *inside* a container, not on the host" reason as the responder # above, and with a second one: putting it in the core bundle would place diff --git a/swarm-metrics-auth/Cargo.toml b/swarm-metrics-auth/Cargo.toml new file mode 100644 index 00000000..6d90939f --- /dev/null +++ b/swarm-metrics-auth/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "swarm-metrics-auth" +version.workspace = true +edition.workspace = true + +[[bin]] +name = "swarm-metrics-auth" +path = "src/main.rs" + +[dependencies] +anyhow.workspace = true +axum.workspace = true +clap.workspace = true +serde.workspace = true +# Comparing the presented bearer with the stored one. +subtle.workspace = true +# Only for the rustls provider the store client needs. +swarm-queue-client.workspace = true +swarm-secret-client.workspace = true +tokio.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true + +[lints] +workspace = true diff --git a/swarm-metrics-auth/src/main.rs b/swarm-metrics-auth/src/main.rs new file mode 100644 index 00000000..41076d78 --- /dev/null +++ b/swarm-metrics-auth/src/main.rs @@ -0,0 +1,354 @@ +//! The metrics store gateway's `auth_request` responder for Grafana. +//! +//! nginx sends it every request to the store's query API that carries a +//! `Bearer` (`swarm-victoriametrics.nix`). It answers 200 when the bearer is +//! the value stored at `--store-path`, which Grafana's host creates and its +//! datasource presents, and 401 otherwise, store failures included: +//! `auth_request` turns any other status into a 500 for the caller. +//! +//! It reads the store under its own certificate, whose policy reaches that one +//! path. It serves a unix socket whose directory only the gateway's nginx may +//! enter. +//! +//! The stored value is kept for `--cache-ttl` seconds, and no request reaches +//! the store within that window: a replaced bearer is refused, and the one it +//! replaced still accepted, for up to that long. Lookups are serialised, so a +//! burst of requests against an expired copy costs one. + +use std::future::Future; +use std::os::unix::fs::PermissionsExt as _; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use anyhow::Context as _; +use axum::extract::State; +use axum::http::{HeaderMap, StatusCode, header}; +use clap::Parser; +use subtle::ConstantTimeEq as _; +use swarm_secret_client::client::{DEFAULT_CERT_MOUNT, Settings}; + +/// How long one store lookup may take before the request is denied. Well +/// under nginx's 60s `proxy_read_timeout`, so a slow store reads as a 401 +/// rather than a gateway timeout. +const LOOKUP_TIMEOUT: Duration = Duration::from_secs(5); + +#[derive(Parser)] +#[command(about = "auth_request responder: checks Grafana's metrics bearer against the store")] +struct Args { + /// Unix socket to serve on. + #[arg(long)] + socket: PathBuf, + /// Cert-auth role this responder logs in to the store as. + #[arg(long, default_value = "swarm-metrics-auth")] + store_cert_role: String, + /// KV path, under the `secret` mount, holding the bearer in its `value` + /// field. + #[arg(long)] + store_path: String, + /// Seconds the stored bearer is used without reading the store again; + /// 0 reads it on every request. + #[arg(long, default_value_t = 60)] + cache_ttl: u64, +} + +/// Where the stored bearer comes from. +trait BearerSource { + /// The stored bearer, `None` when nothing is stored, or an error when the + /// store could not answer. + fn lookup(&self) -> impl Future>> + Send; +} + +/// The object at the bearer's path. `bao kv put … value=-` on Grafana's host +/// writes this field. +#[derive(serde::Deserialize)] +struct Stored { + value: String, +} + +/// The swarm secret store, reached with the responder's own certificate. +struct Store { + settings: Settings, + cert_role: String, + path: String, +} + +impl BearerSource for Store { + /// Logs in per lookup, so no token is left to expire inside a long-lived + /// process. + async fn lookup(&self) -> anyhow::Result> { + let store = swarm_secret_client::SecretStore::connect( + &self.settings, + &self.cert_role, + DEFAULT_CERT_MOUNT, + ) + .await + .context("logging in to the secret store")?; + let stored: Option = store + .read_optional(&self.path) + .await + .with_context(|| format!("reading {}", self.path))?; + Ok(stored.map(|s| s.value)) + } +} + +struct Cached { + value: String, + read_at: Instant, +} + +struct Responder { + source: S, + ttl: Duration, + cached: tokio::sync::Mutex>, +} + +/// The answer for one subrequest's `Authorization` header, at `now`. +async fn decide( + responder: &Responder, + authorization: Option<&str>, + now: Instant, +) -> StatusCode { + let Some(bearer) = authorization.and_then(|v| v.strip_prefix("Bearer ")) else { + tracing::warn!("no bearer token; denying"); + return StatusCode::UNAUTHORIZED; + }; + // Held across the lookup, so concurrent requests wait for one read + // rather than each starting their own. + let mut cached = responder.cached.lock().await; + let fresh = cached + .as_ref() + .is_some_and(|c| now.saturating_duration_since(c.read_at) < responder.ttl); + if !fresh { + match tokio::time::timeout(LOOKUP_TIMEOUT, responder.source.lookup()).await { + Ok(Ok(Some(value))) if !value.is_empty() => { + *cached = Some(Cached { + value, + read_at: now, + }); + } + Ok(Ok(_)) => { + tracing::warn!("no bearer is stored; denying"); + return StatusCode::UNAUTHORIZED; + } + Ok(Err(e)) => { + tracing::warn!(error = format!("{e:#}"), "bearer lookup failed; denying"); + return StatusCode::UNAUTHORIZED; + } + Err(_) => { + tracing::warn!("bearer lookup timed out; denying"); + return StatusCode::UNAUTHORIZED; + } + } + } + let Some(stored) = cached.as_ref() else { + return StatusCode::UNAUTHORIZED; + }; + // Constant time, so the time to refuse says nothing about how much of the + // bearer was right. + if bool::from(stored.value.as_bytes().ct_eq(bearer.as_bytes())) { + StatusCode::OK + } else { + tracing::warn!("bearer does not match the stored one; denying"); + StatusCode::UNAUTHORIZED + } +} + +async fn handle(State(responder): State>>, headers: HeaderMap) -> StatusCode { + let authorization = headers + .get(header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()); + decide(&responder, authorization, Instant::now()).await +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + swarm_queue_client::install_crypto_provider(); + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + // stdout is journald, which keeps ANSI escapes as raw bytes. + .with_ansi(false) + .init(); + let args = Args::parse(); + + let settings = Settings::from_env() + .context("reading the secret store's address and this responder's identity")?; + let responder = Arc::new(Responder { + source: Store { + settings, + cert_role: args.store_cert_role, + path: args.store_path, + }, + ttl: Duration::from_secs(args.cache_ttl), + cached: tokio::sync::Mutex::new(None), + }); + + // `RuntimeDirectoryPreserve=yes` can leave the previous run's socket; the + // directory is this unit's alone, so unlinking it is safe. + if let Err(e) = std::fs::remove_file(&args.socket) + && e.kind() != std::io::ErrorKind::NotFound + { + return Err(e) + .with_context(|| format!("clearing stale socket at {}", args.socket.display())); + } + let listener = tokio::net::UnixListener::bind(&args.socket) + .with_context(|| format!("binding {}", args.socket.display()))?; + // nginx is another user and `connect(2)` needs write. The directory is + // the access control, not this mode. + std::fs::set_permissions(&args.socket, std::fs::Permissions::from_mode(0o666)) + .with_context(|| format!("chmod {}", args.socket.display()))?; + tracing::info!(socket = %args.socket.display(), "swarm-metrics-auth listening"); + + let app = axum::Router::new().fallback(handle).with_state(responder); + axum::serve(listener, app).await.context("serving") +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex; + use std::sync::atomic::{AtomicUsize, Ordering}; + + /// A store answering `stored` (or failing when it is `Err`), counting + /// lookups. + struct Fake { + stored: Mutex, ()>>, + lookups: AtomicUsize, + } + + impl Fake { + fn holding(value: &str) -> Self { + Self { + stored: Mutex::new(Ok(Some(value.to_owned()))), + lookups: AtomicUsize::new(0), + } + } + + fn set(&self, stored: Result, ()>) { + *self.stored.lock().unwrap() = stored.map(|v| v.map(str::to_owned)); + } + + fn lookups(&self) -> usize { + self.lookups.load(Ordering::SeqCst) + } + } + + impl BearerSource for &Fake { + async fn lookup(&self) -> anyhow::Result> { + self.lookups.fetch_add(1, Ordering::SeqCst); + self.stored + .lock() + .unwrap() + .clone() + .map_err(|()| anyhow::anyhow!("store unreachable")) + } + } + + const TTL: Duration = Duration::from_mins(1); + const SECRET: &str = "0f1e2d3c4b5a"; + const GOOD: &str = "Bearer 0f1e2d3c4b5a"; + + fn responder(store: &Fake) -> Responder<&Fake> { + Responder { + source: store, + ttl: TTL, + cached: tokio::sync::Mutex::new(None), + } + } + + #[tokio::test] + async fn the_stored_bearer_is_200() { + let store = Fake::holding(SECRET); + assert_eq!( + decide(&responder(&store), Some(GOOD), Instant::now()).await, + StatusCode::OK + ); + } + + #[tokio::test] + async fn everything_else_is_401() { + let store = Fake::holding(SECRET); + let r = responder(&store); + for authorization in [ + None, + Some(SECRET), + Some("Basic 0f1e2d3c4b5a"), + Some("Bearer 0f1e2d3c4b5b"), + Some("Bearer 0f1e2d3c4b5"), + Some("Bearer "), + Some("Bearer authelia_at_abc.def"), + ] { + assert_eq!( + decide(&r, authorization, Instant::now()).await, + StatusCode::UNAUTHORIZED, + "{authorization:?}" + ); + } + } + + /// Nothing stored, an empty value, and a store that cannot answer are + /// all denials, whatever the bearer. + #[tokio::test] + async fn no_usable_stored_value_is_401() { + let store = Fake::holding(SECRET); + let r = responder(&store); + for stored in [Ok(None), Ok(Some("")), Err(())] { + store.set(stored); + assert_eq!( + decide(&r, Some(GOOD), Instant::now()).await, + StatusCode::UNAUTHORIZED, + "{stored:?}" + ); + assert_eq!( + decide(&r, Some("Bearer "), Instant::now()).await, + StatusCode::UNAUTHORIZED, + "{stored:?}" + ); + } + } + + /// Within the TTL no request reaches the store, a wrong bearer included, + /// and a replacement the store already holds goes unseen; after it the + /// store is read again. + #[tokio::test] + async fn the_stored_value_is_reused_until_the_ttl() { + let store = Fake::holding(SECRET); + let r = responder(&store); + let t0 = Instant::now(); + assert_eq!(decide(&r, Some(GOOD), t0).await, StatusCode::OK); + store.set(Ok(Some("replaced"))); + let within = t0 + Duration::from_secs(59); + assert_eq!(decide(&r, Some(GOOD), within).await, StatusCode::OK); + assert_eq!( + decide(&r, Some("Bearer replaced"), within).await, + StatusCode::UNAUTHORIZED + ); + assert_eq!(store.lookups(), 1); + assert_eq!( + decide(&r, Some(GOOD), t0 + TTL).await, + StatusCode::UNAUTHORIZED + ); + assert_eq!( + decide(&r, Some("Bearer replaced"), t0 + TTL).await, + StatusCode::OK + ); + assert_eq!(store.lookups(), 2); + } + + /// A failed read leaves nothing usable cached, so the next request reads + /// the store again rather than waiting out a TTL. + #[tokio::test] + async fn a_failed_lookup_is_retried_on_the_next_request() { + let store = Fake::holding(SECRET); + store.set(Err(())); + let r = responder(&store); + let t0 = Instant::now(); + assert_eq!(decide(&r, Some(GOOD), t0).await, StatusCode::UNAUTHORIZED); + store.set(Ok(Some(SECRET))); + assert_eq!(decide(&r, Some(GOOD), t0).await, StatusCode::OK); + assert_eq!(store.lookups(), 2); + } +}