grafana: query the log store with the metrics bearer, by its swarm name
Grafana's VictoriaLogs datasource now targets https://logs.<swarm> and sends the same static bearer as the metrics datasource. The log store's vhost routes a 64-hex Authorization bearer to a second swarm-metrics-auth instance (swarm-logs-grafana-auth, cert role swarm-logs-auth, which gains a read on the bearer's store path). swarm-agent. tokens and everything else keep their routes. Part of #3742 (DoD 3, logs).
This commit is contained in:
atlas
mara
parent
8ebfe34531
commit
eff5c43bdc
9 changed files with 200 additions and 52 deletions
|
|
@ -1003,10 +1003,10 @@ let
|
|||
# of `adminPasswordStorePath` in ./swarm-grafana.nix.
|
||||
grafanaAdminLeaf = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/admin/password";
|
||||
|
||||
# The bearer Grafana's metrics datasource presents at the metrics store.
|
||||
# Grafana's leaf creates it, the store's responder reads it. The nix half of
|
||||
# The bearer Grafana's datasources present at the metrics and log stores.
|
||||
# Grafana's leaf creates it, the stores' responders read it. The nix half of
|
||||
# `metricsBearerStorePath` in ./swarm-grafana.nix and of `--store-path` in
|
||||
# ./swarm-victoriametrics.nix.
|
||||
# ./swarm-victoriametrics.nix and ./swarm-victorialogs.nix.
|
||||
grafanaMetricsBearerLeaf = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer";
|
||||
|
||||
createStanza = path: ''
|
||||
|
|
@ -1015,14 +1015,17 @@ let
|
|||
}
|
||||
'';
|
||||
|
||||
# The log store's agent auth responder, which checks the secret an agent
|
||||
# presents against the one stored for it: the same `+` leaf as the queue
|
||||
# responder's, and nothing else.
|
||||
# The log store's auth responders: the agent one checks the secret an agent
|
||||
# presents against the one stored for it (the same `+` leaf as the queue
|
||||
# responder's), the other checks the bearer Grafana presents against
|
||||
# Grafana's metrics bearer. Nothing else.
|
||||
logsAuthReaders = [
|
||||
{
|
||||
name = "swarm-logs-auth";
|
||||
cn = baoDeploy.logsAuthCommonName;
|
||||
policyText = readStanza "${credentialMountPath}/data/swarm/agents/+/queue";
|
||||
policyText =
|
||||
readStanza "${credentialMountPath}/data/swarm/agents/+/queue"
|
||||
+ readStanza "${credentialMountPath}/data/${grafanaMetricsBearerLeaf}";
|
||||
}
|
||||
];
|
||||
|
||||
|
|
|
|||
|
|
@ -274,8 +274,10 @@ let
|
|||
|
||||
# The bearer the metrics datasource presents at the metrics store's query
|
||||
# API, which `swarm-metrics-auth` on the store's host checks against the
|
||||
# same store path. Minted once by `swarm-bao-grafana-metrics-bearer` below,
|
||||
# under the same grant as the admin password. The file holds the whole
|
||||
# same store path, and the logs datasource at the log store's query API,
|
||||
# which a second `swarm-metrics-auth` instance in ./swarm-victorialogs.nix
|
||||
# checks against it. Minted once by `swarm-bao-grafana-metrics-bearer`
|
||||
# below, under the same grant as the admin password. The file holds the whole
|
||||
# header value, `Bearer <value>`: nixpkgs accepts a `secureJsonData` entry
|
||||
# only when it is exactly one `$__file{}`.
|
||||
metricsBearerStorePath = "secret/swarm/services/${cfg.oidc.clientId}/metrics/bearer";
|
||||
|
|
@ -371,16 +373,20 @@ in
|
|||
|
||||
logsDatasourceUrl = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "http://127.0.0.1:${toString hyperhiveCfg.swarm.victorialogs.port}";
|
||||
defaultText = lib.literalExpression ''"http://127.0.0.1:''${toString services.hyperhive.swarm.victorialogs.port}"'';
|
||||
default = "https://${hyperhiveCfg.swarm.victorialogs.domain}";
|
||||
defaultText = lib.literalExpression ''"https://''${services.hyperhive.swarm.victorialogs.domain}"'';
|
||||
description = ''
|
||||
Where the provisioned logs datasource points. Defaults to the log
|
||||
store on this host, which binds loopback, so this default reaches
|
||||
it only where Grafana and the store share a host.
|
||||
store's swarm name, served by the gateway of whichever host runs the
|
||||
store, so Grafana and the store need not share a host.
|
||||
|
||||
Provisioned unconditionally, like the metrics datasource — the store
|
||||
being off is a deployment choice rather than a reason to withhold the
|
||||
connection, and an operator whose logs live elsewhere sets this.
|
||||
|
||||
The datasource sends the same bearer as the metrics one, which the
|
||||
store's `/select/logsql/` route checks. Whatever URL this names
|
||||
receives that bearer.
|
||||
'';
|
||||
};
|
||||
|
||||
|
|
@ -1316,7 +1322,11 @@ in
|
|||
# not in the pipeline.
|
||||
jsonData = {
|
||||
inherit logLevelRules;
|
||||
httpHeaderName1 = "Authorization";
|
||||
};
|
||||
# The metrics datasource's bearer; the log store's vhost
|
||||
# accepts it on `/select/logsql/`.
|
||||
secureJsonData.httpHeaderValue1 = "$__file{${metricsBearerPath}}";
|
||||
}
|
||||
];
|
||||
};
|
||||
|
|
|
|||
|
|
@ -68,6 +68,15 @@ let
|
|||
};
|
||||
baoCfg = hyperhiveCfg.swarm.bao;
|
||||
|
||||
# Grafana's bearer, checked by a second `swarm-metrics-auth` instance. It
|
||||
# reads the store with the same leaf as `swarm-logs-auth`, which is granted
|
||||
# the one extra path below. The same leaf as `grafanaBearerStorePath` in
|
||||
# ./swarm-victoriametrics.nix and `grafanaMetricsBearerLeaf` in ./swarm-bao.nix.
|
||||
grafanaAuthUnit = "swarm-logs-grafana-auth";
|
||||
grafanaAuthSocket = "/run/${grafanaAuthUnit}/auth.sock";
|
||||
grafanaAuthCredential = id: "/run/credentials/${grafanaAuthUnit}.service/${id}";
|
||||
grafanaBearerStorePath = "swarm/services/${hyperhiveCfg.swarm.grafana.oidc.clientId}/metrics/bearer";
|
||||
|
||||
# Shared host netns, like every sibling swarm container: Grafana reaches
|
||||
# this at 127.0.0.1:<port>.
|
||||
privateNetwork = false;
|
||||
|
|
@ -210,6 +219,7 @@ in
|
|||
services.nginx.appendHttpConfig = ''
|
||||
map $http_authorization $swarm_logs_auth_target {
|
||||
"~^Bearer swarm-agent\." /__swarm_logs_agent_auth;
|
||||
"~^Bearer [0-9a-f]{64}$" /__swarm_logs_grafana_auth;
|
||||
default /__hive_authelia;
|
||||
}
|
||||
limit_req_zone $binary_remote_addr zone=swarm_logs_agent_auth:1m rate=${deployCfg.victorialogs.authRate};
|
||||
|
|
@ -289,9 +299,9 @@ in
|
|||
# permissions are a later thing, and this location is where one
|
||||
# attaches when it exists.
|
||||
#
|
||||
# Two checks: an agent's own `swarm-agent.` bearer goes to
|
||||
# `swarm-logs-auth`, anything else to authelia (`appendHttpConfig`
|
||||
# above picks).
|
||||
# Three checks: an agent's own `swarm-agent.` bearer goes to
|
||||
# `swarm-logs-auth`, Grafana's 64-hex bearer to `swarm-metrics-auth`,
|
||||
# anything else to authelia (`appendHttpConfig` above picks).
|
||||
"^~ /select/logsql/" = {
|
||||
# No URI part, so the request path and its query string reach
|
||||
# the store as the caller sent them.
|
||||
|
|
@ -345,6 +355,18 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
# Grafana's bearer, 64 hex digits (`openssl rand -hex 32`), checked by
|
||||
# `swarm-metrics-auth`. The responder reads the store at most once per
|
||||
# its cache TTL whatever the bearers, so no rate limit sits in front.
|
||||
"= /__swarm_logs_grafana_auth" = {
|
||||
proxyPass = "http://unix:${grafanaAuthSocket}:/";
|
||||
extraConfig = ''
|
||||
internal;
|
||||
proxy_pass_request_body off;
|
||||
proxy_set_header Content-Length "";
|
||||
'';
|
||||
};
|
||||
|
||||
# The subrequest itself — same target, same header set, same
|
||||
# reasoning as `swarm-ui.nix`'s own copy (measured against the
|
||||
# pinned authelia binary, not copied from an example).
|
||||
|
|
@ -430,6 +452,57 @@ in
|
|||
};
|
||||
};
|
||||
|
||||
# Grafana's check on `/select/logsql/`: the metrics store's responder
|
||||
# binary, pointed at the bearer Grafana's datasources share.
|
||||
users.users.${grafanaAuthUnit} = {
|
||||
isSystemUser = true;
|
||||
group = grafanaAuthUnit;
|
||||
description = "swarm log store auth_request responder for Grafana's bearer";
|
||||
};
|
||||
users.groups.${grafanaAuthUnit} = { };
|
||||
|
||||
systemd.services.${grafanaAuthUnit} = {
|
||||
description = "swarm log store auth_request responder for Grafana's bearer";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [
|
||||
"network.target"
|
||||
"swarm-bao-pki.service"
|
||||
];
|
||||
serviceConfig = {
|
||||
ExecStart = lib.concatStringsSep " " [
|
||||
"${deployCfg.victoriametrics.authPackage}/bin/swarm-metrics-auth"
|
||||
"--socket ${grafanaAuthSocket}"
|
||||
"--store-cert-role swarm-logs-auth"
|
||||
"--store-path ${grafanaBearerStorePath}"
|
||||
];
|
||||
LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") logsAuthStoreFiles;
|
||||
# ⚠️ A static user, never `DynamicUser`: see the `swarm-logs-auth` unit above.
|
||||
User = grafanaAuthUnit;
|
||||
Group = grafanaAuthUnit;
|
||||
RuntimeDirectory = grafanaAuthUnit;
|
||||
RuntimeDirectoryMode = "0751";
|
||||
RuntimeDirectoryPreserve = "yes";
|
||||
Restart = "on-failure";
|
||||
RestartSec = "5s";
|
||||
SyslogIdentifier = grafanaAuthUnit;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
PrivateTmp = "disconnected";
|
||||
PrivateDevices = true;
|
||||
NoNewPrivileges = true;
|
||||
RestrictSUIDSGID = true;
|
||||
RemoveIPC = true;
|
||||
};
|
||||
environment = {
|
||||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
BAO_CLIENT_CERT = grafanaAuthCredential "bao-client.pem";
|
||||
BAO_CLIENT_KEY = grafanaAuthCredential "bao-client-key.pem";
|
||||
}
|
||||
// lib.optionalAttrs (logsAuthStoreFiles ? "bao-ca.pem") {
|
||||
BAO_CACERT = grafanaAuthCredential "bao-ca.pem";
|
||||
};
|
||||
};
|
||||
|
||||
containers.${cfg.machine} = {
|
||||
autoStart = true;
|
||||
ephemeral = false;
|
||||
|
|
|
|||
Loading…
Reference in a new issue