e92f118bda
swarm-metrics reads the swarm's VictoriaMetrics from inside an agent
container: query, query-range, series, labels and label-values, each one
GET against the store's Prometheus read API through the gateway, printing
the store's JSON. It authenticates the way swarm-logs does, with the
agent's own queue secret read from bao under the agent's leaf, sent as
a swarm-agent bearer.
On the metrics store's host the read API's auth map gains an agent arm
ahead of the Grafana one. It routes to swarm-metrics-agent-auth, a
swarm-logs-auth instance logged in under the existing swarm-metrics-auth
leaf, whose bao grant now includes swarm/agents/+/queue read. The check is
rate-limited per address like the log store's
(deploy.victoriametrics.agentAuth{Rate,Burst,VerdictTtl}).
nix/agent-modules/metrics.nix installs the wrapped binary when the hive
forwards HIVE_AGENT_METRICS_URL (from swarm.victoriametrics.domain) and a
store address, the same gate as swarm-logs. Adds the base:swarm-metrics
skill and the CLI reference with its freshness check.
Fixes #5395
279 lines
12 KiB
Nix
279 lines
12 KiB
Nix
# `checks.module-eval-agent-queue-bao` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
shared,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
agent
|
|
agentWith
|
|
runGroup
|
|
agentHarness
|
|
;
|
|
|
|
# The agent side of the swarm queue. Both coordinates set is the only state
|
|
# in which the harness unit declares a credential at all, so the pair and
|
|
# the empty fixture beside it are the two arms worth having.
|
|
agentQueue = agent {
|
|
queue.natsUrl = "nats://10.42.0.1:4222";
|
|
queue.tokenEndpoint = "https://auth.t.local/api/oidc/token";
|
|
};
|
|
|
|
agentNoQueue = shared.agentBare;
|
|
|
|
# The agent side of the swarm secret store. The address is the whole switch —
|
|
# it is both what generates the login check and what that check points at —
|
|
# so it and the empty fixture beside it are the two arms worth having.
|
|
agentBao = shared.agentWithBao;
|
|
|
|
agentNoBao = shared.agentBare;
|
|
|
|
# `swarm-logs` and `swarm-metrics` authenticate with the agent's queue secret
|
|
# from the store, so the store address gates each beside its store URL.
|
|
logsUrl = "https://logs.t.local/select/logsql/query";
|
|
metricsUrl = "https://metrics.t.local";
|
|
agentLogs = agentWith {
|
|
services.hyperhive.agent.bao.addr = "https://bao.t.local:8200";
|
|
services.hyperhive.agent.logs.queryUrl = logsUrl;
|
|
services.hyperhive.agent.metrics.url = metricsUrl;
|
|
};
|
|
agentLogsNoBao = agentWith {
|
|
services.hyperhive.agent.logs.queryUrl = logsUrl;
|
|
services.hyperhive.agent.metrics.url = metricsUrl;
|
|
};
|
|
configuredIn =
|
|
name: machine: lib.findFirst (p: (p.name or "") == name) null machine.environment.systemPackages;
|
|
swarmLogsIn = configuredIn "swarm-logs-configured";
|
|
swarmMetricsIn = configuredIn "swarm-metrics-configured";
|
|
|
|
agentBaoIdentity = machine: machine.systemd.services.hive-agent-bao-identity;
|
|
cases = [
|
|
{
|
|
# The wrapper names the store identity ./queue-identity.nix gives the
|
|
# harness, by path, and nothing of the hive's shared OIDC client.
|
|
name = "swarm-logs is wrapped with the agent's own store identity";
|
|
ok =
|
|
let
|
|
w = (swarmLogsIn agentLogs).buildCommand;
|
|
a = agentLogs.services.hyperhive.agent;
|
|
set = k: v: lib.hasInfix "--set ${k} ${lib.escapeShellArg v}" w;
|
|
in
|
|
set "HIVE_AGENT_LOGS_QUERY_URL" logsUrl
|
|
&& set "HIVE_AGENT_NAME" a.user.name
|
|
&& set "BAO_ADDR" "https://bao.t.local:8200"
|
|
&& set "BAO_CLIENT_CERT" a.bao.clientCertFile
|
|
&& set "BAO_CLIENT_KEY" a.bao.clientKeyFile
|
|
&& set "BAO_CACERT" "/run/credentials/hive-agent.service/hive-agent-bao-server-ca"
|
|
&& !(lib.hasInfix "OIDC" w);
|
|
}
|
|
{
|
|
# A query URL with nothing to authenticate as installs nothing.
|
|
name = "an agent with a log store but no secret store gets no swarm-logs";
|
|
ok = swarmLogsIn agentLogsNoBao == null && swarmLogsIn agentLogs != null;
|
|
}
|
|
{
|
|
# The same store identity as `swarm-logs`, and the store's origin.
|
|
name = "swarm-metrics is wrapped with the agent's own store identity";
|
|
ok =
|
|
let
|
|
w = (swarmMetricsIn agentLogs).buildCommand;
|
|
a = agentLogs.services.hyperhive.agent;
|
|
set = k: v: lib.hasInfix "--set ${k} ${lib.escapeShellArg v}" w;
|
|
in
|
|
set "HIVE_AGENT_METRICS_URL" metricsUrl
|
|
&& set "HIVE_AGENT_NAME" a.user.name
|
|
&& set "BAO_ADDR" "https://bao.t.local:8200"
|
|
&& set "BAO_CLIENT_CERT" a.bao.clientCertFile
|
|
&& set "BAO_CLIENT_KEY" a.bao.clientKeyFile
|
|
&& set "BAO_CACERT" "/run/credentials/hive-agent.service/hive-agent-bao-server-ca";
|
|
}
|
|
{
|
|
# A metrics URL with nothing to authenticate as installs nothing.
|
|
name = "an agent with a metrics store but no secret store gets no swarm-metrics";
|
|
ok = swarmMetricsIn agentLogsNoBao == null && swarmMetricsIn agentLogs != null;
|
|
}
|
|
{
|
|
# Both ids or neither: the secret authenticates nobody without the id it
|
|
# belongs to, and `swarm-logs` reads both.
|
|
name = "an agent with queue coordinates imports both halves of its credential";
|
|
ok =
|
|
let
|
|
c = (agentHarness agentQueue).serviceConfig.LoadCredential;
|
|
in
|
|
builtins.elem "hive-queue-agent-secret" c && builtins.elem "hive-queue-agent-client-id" c;
|
|
}
|
|
{
|
|
# An agent built before its hive was handed the queue's address. It
|
|
# must declare nothing rather than name a credential that never
|
|
# arrives.
|
|
name = "an agent with no queue coordinates declares no credential";
|
|
ok = !((agentHarness agentNoQueue).serviceConfig ? LoadCredential);
|
|
}
|
|
{
|
|
# The harness exits when it cannot reach the queue. It has to come back
|
|
# however long the outage lasts, and slowly enough that every agent of a
|
|
# swarm restarting at once does not hammer the store and the queue.
|
|
name = "the harness restarts every 30s with no start limit";
|
|
ok =
|
|
let
|
|
u = agentHarness agentQueue;
|
|
in
|
|
u.serviceConfig.Restart == "on-failure"
|
|
&& u.serviceConfig.RestartSec == 30
|
|
&& u.startLimitIntervalSec == 0;
|
|
}
|
|
{
|
|
# The three ids `hive_c0re::lifecycle::agent_identity` forwards under.
|
|
# Neither end can discover the other's spelling, and a mismatch is a
|
|
# credential that is simply not there — which this unit then reports as
|
|
# a hive that delivered nothing.
|
|
name = "an agent with the store enabled imports every half of its identity";
|
|
ok =
|
|
let
|
|
c = (agentBaoIdentity agentBao).serviceConfig.LoadCredential;
|
|
in
|
|
builtins.elem "hive-agent-bao-cert" c
|
|
&& builtins.elem "hive-agent-bao-key" c
|
|
&& builtins.elem "hive-agent-bao-server-ca" c;
|
|
}
|
|
{
|
|
# The live pair sits in the unit's own runtime directory, which outlives
|
|
# each run so the timer's next run logs in with what the last one
|
|
# wrote. The address is the option's value rather than a literal that
|
|
# agrees with it today.
|
|
name = "the identity check presents the live pair it keeps in its runtime directory";
|
|
ok =
|
|
let
|
|
u = agentBaoIdentity agentBao;
|
|
e = u.environment;
|
|
in
|
|
e.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert"
|
|
&& e.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key"
|
|
&& u.serviceConfig.RuntimeDirectory == "hive-agent-bao-identity"
|
|
&& u.serviceConfig.RuntimeDirectoryPreserve == "yes"
|
|
&& e.BAO_ADDR == agentBao.services.hyperhive.agent.bao.addr;
|
|
}
|
|
{
|
|
# A leaf the controller re-issues reaches a running container only
|
|
# through this re-run, and an active unit cannot be started again.
|
|
name = "the identity check re-runs on a timer";
|
|
ok =
|
|
(agentBaoIdentity agentBao).serviceConfig.RemainAfterExit == false
|
|
&& agentBao.systemd.timers.hive-agent-bao-identity.timerConfig.OnUnitInactiveSec == "10min";
|
|
}
|
|
{
|
|
# Same 403-not-a-miss reason as the hive-side readers: the path
|
|
# `swarm_secret_client::mtls::identity_path` builds is the one this
|
|
# agent's own policy stanza covers, and a path outside it is refused
|
|
# however correct it looks. Built from the agent's own name rather than
|
|
# from a literal, because the name is what makes it this agent's path
|
|
# and not some other agent's.
|
|
name = "the identity check reads the agent's own path";
|
|
ok =
|
|
let
|
|
m = agentBao;
|
|
name = m.services.hyperhive.agent.user.name;
|
|
in
|
|
lib.hasInfix "secret/swarm/agents/${name}/bao-mtls" (agentBaoIdentity m).script;
|
|
}
|
|
{
|
|
# The whole point of the unit, and the thing a quieter default would
|
|
# undo: every arm of the check ends the unit non-zero, so an agent that
|
|
# cannot authenticate as itself says so at boot instead of at whichever
|
|
# pull needed the store first.
|
|
name = "the identity check fails the unit rather than degrading";
|
|
ok =
|
|
let
|
|
u = agentBaoIdentity agentBao;
|
|
in
|
|
lib.hasInfix "exit 1" u.script
|
|
&& !(lib.hasInfix "exit 0" u.script)
|
|
&& u.serviceConfig.Restart == "on-failure";
|
|
}
|
|
{
|
|
# Nothing about the identity may be printed, and the read-back is where
|
|
# that could slip: `bao kv get` on this path answers with certificate
|
|
# material and a private key. The answer goes to a file in the unit's
|
|
# own `0700` directory.
|
|
#
|
|
# The path goes through `lib.escapeShellArg` here for the same reason the
|
|
# module passes it through one — that helper decides whether an argument
|
|
# needs quotes at all, and this one (only `[a-z0-9/-]`) comes back bare.
|
|
# Spelling the quotes in by hand asserts a rendering nixpkgs chooses
|
|
# rather than the redirect this property is about.
|
|
name = "the identity check writes what it reads back to a file";
|
|
ok =
|
|
let
|
|
m = agentBao;
|
|
name = m.services.hyperhive.agent.user.name;
|
|
arg = lib.escapeShellArg "secret/swarm/agents/${name}/bao-mtls";
|
|
in
|
|
lib.hasInfix "bao kv get -format=json ${arg} >\"$raw\"" (agentBaoIdentity m).script;
|
|
}
|
|
{
|
|
# The absence arm, and what makes the four above able to fail. An agent
|
|
# whose swarm never minted an identity has nothing to log in with, and a
|
|
# failed unit at every boot would be the loudest possible statement
|
|
# about a deployment that never asked for one.
|
|
name = "an agent told no store address runs no identity check";
|
|
ok = !(agentNoBao.systemd.services ? hive-agent-bao-identity);
|
|
}
|
|
{
|
|
# The harness reads its per-agent queue secret from the store itself,
|
|
# under the same store identity the check above keeps current, because
|
|
# there is one identity per agent.
|
|
name = "the harness presents the agent's own store identity";
|
|
ok =
|
|
let
|
|
u = agentHarness agentBao;
|
|
in
|
|
u.environment.BAO_CLIENT_CERT == "/run/hive-agent-bao-identity/current/cert"
|
|
&& u.environment.BAO_CLIENT_KEY == "/run/hive-agent-bao-identity/current/key"
|
|
&& builtins.elem "hive-agent-bao-identity.service" u.after
|
|
&& u.environment.BAO_ADDR == agentBao.services.hyperhive.agent.bao.addr;
|
|
}
|
|
{
|
|
# `swarm_secret_client` builds both the store path and the cert-auth role
|
|
# from this name, so it has to be the name the credential was minted
|
|
# under.
|
|
name = "the harness is told the agent name its queue secret is minted under";
|
|
ok =
|
|
(agentHarness agentBao).environment.HIVE_AGENT_NAME == agentBao.services.hyperhive.agent.user.name;
|
|
}
|
|
{
|
|
# The per-agent queue secret lives in the harness's memory only: no unit
|
|
# fetches it to a file, and the harness is handed no file to read it
|
|
# from.
|
|
name = "no unit writes the per-agent queue secret to disk";
|
|
ok =
|
|
!(agentBao.systemd.services ? hive-agent-queue-credential)
|
|
&& !((agentHarness agentBao).environment ? HIVE_AGENT_QUEUE_AGENT_SECRET_FILE);
|
|
}
|
|
{
|
|
# The absence arm. An agent whose swarm gave it no store has nothing to
|
|
# log in with, so the harness is handed no store coordinates at all.
|
|
name = "an agent told no store address hands the harness no store identity";
|
|
ok =
|
|
let
|
|
u = agentHarness agentNoBao;
|
|
in
|
|
!(u.environment ? BAO_ADDR)
|
|
&& !(u.environment ? HIVE_AGENT_NAME)
|
|
&& !(builtins.elem "hive-agent-bao-server-ca" (u.serviceConfig.LoadCredential or [ ]));
|
|
}
|
|
];
|
|
in
|
|
runGroup "agent-queue-bao" cases
|