diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 2e52ccb5..aae9b965 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -164,8 +164,12 @@ Grafana start — see [`secrets.md`](secrets.md). **Where the data comes from.** The swarm's OTEL collector, below. Neither container is reachable except through the gateway: both bind -loopback, and VictoriaMetrics' write endpoint takes no credential, so -the collector is the only intended writer. +loopback, and VictoriaMetrics authenticates nothing itself. The +`metrics.` vhost forwards only the store's read API (the +Prometheus query, series, label and metadata routes, plus the vmui web UI) +without a credential. It forwards the collector's OTLP push only with an +authelia token, and refuses every other path, the store's write and admin +API included, with 403. The bare domain redirects to vmui. ### Logs (VictoriaLogs) diff --git a/nix/host-modules/swarm-victoriametrics.nix b/nix/host-modules/swarm-victoriametrics.nix index 2435a1dd..63164a2b 100644 --- a/nix/host-modules/swarm-victoriametrics.nix +++ b/nix/host-modules/swarm-victoriametrics.nix @@ -25,6 +25,51 @@ let # Shared host netns, like every sibling swarm container: the gateway # reaches this at 127.0.0.1:. privateNetwork = false; + + # The store's read API: the only part of it the vhost serves without a + # credential. Each location's `proxy_pass` carries the path, so the store + # receives the path nginx matched, not the caller's unnormalised spelling. + # + # Grafana's Prometheus datasource reads the query, series, label, metadata, + # buildinfo, exemplar, rules and alerts routes; the store answers the last + # three with fixed placeholders. vmui is the static files under `/vmui/` + # plus the query API and the remaining read-only routes here. Paths read off + # the pinned build's `app/vmselect/main.go` and vmui's `src/`. + readPaths = [ + "/api/v1/query" + "/api/v1/query_range" + "/api/v1/query_exemplars" + "/api/v1/series" + "/api/v1/labels" + "/api/v1/metadata" + "/api/v1/rules" + "/api/v1/alerts" + "/api/v1/status/buildinfo" + "/api/v1/export" + "/api/v1/export/csv" + "/api/v1/status/tsdb" + "/api/v1/status/metric_names_stats" + "/api/v1/status/top_queries" + "/api/v1/status/active_queries" + "/expand-with-exprs" + "/prettify-query" + "/metric-relabel-debug" + "/vmui" + ]; + # The store serves only `/values` under `/api/v1/label/`, and only + # static files and UI config under `/vmui/`. + readPrefixes = [ + "/api/v1/label/" + "/vmui/" + ]; + readLocations = + let + to = path: { proxyPass = "http://127.0.0.1:${toString cfg.port}${path}"; }; + in + lib.listToAttrs ( + map (p: lib.nameValuePair "= ${p}" (to p)) readPaths + ++ map (p: lib.nameValuePair "^~ ${p}" (to p)) readPrefixes + ); in { # What the store IS from any hive's point of view, the name it answers on and @@ -90,14 +135,19 @@ in services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // { listen = gatewayCfg.lib.listen; extraConfig = gatewayCfg.lib.securityHeaders; - locations = { - "/" = { - proxyPass = "http://127.0.0.1:${toString cfg.port}/"; - }; + locations = readLocations // { + # Nothing reaches the store unless a location here names it. The + # store serves writes and admin calls (`/api/v1/import`, + # `/api/v1/admin/tsdb/delete_series`, ...) on the same listener as + # its queries, and authenticates none of them. + "/".return = "403"; + + # The swarm UI's quick link opens the bare domain. The store's own + # index page links mostly to paths refused above, so send it to vmui. + "= /".return = "302 /vmui/"; # The swarm collector's ingest route, and the only authenticated thing - # on this vhost. `=` so it outranks the `/` prefix above, which would - # otherwise carry these writes with no check at all. + # on this vhost. `=` so it outranks the `/` prefix above. # # ⚠️ No `error_page 401 =302` here, and its absence is the point: a # redirect is right for a browser and wrong for a pusher, which would diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index 986bce99..605897c9 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -425,6 +425,38 @@ let && !(lib.hasInfix "error_page" l) && lib.hasInfix "error_page" browser; } + { + # The store authenticates nothing itself, so every location that + # forwards without `auth_request` is open to anything that can reach the + # domain. The query location is the positive control: it shows the + # filter finds forwarding locations at all. + name = "the metrics vhost forwards only read paths without a credential"; + ok = + let + locs = allLocal.services.nginx.virtualHosts."metrics.t.local".locations; + open = lib.filterAttrs ( + _: l: + l.proxyPass != null + && !(lib.hasInfix "auth_request" l.extraConfig) + && !(lib.hasInfix "internal;" l.extraConfig) + ) locs; + writeWords = [ + "import" + "write" + "admin" + "delete" + "insert" + "internal" + "push" + "prometheus" + "influx" + ]; + in + locs."/".proxyPass == null + && locs."/".return == "403" + && open ? "= /api/v1/query" + && lib.all (n: n != "/" && !(lib.any (w: lib.hasInfix w n) writeWords)) (lib.attrNames open); + } { # The read counterpart to the ingest location: an agent queries the log # store with a bearer token, and the `/` catch-all is the browser's