bb0afcd256
bao's metrics were scraped by the SWARM collector over loopback, via a `swarm.otel.scrapeTargets.bao` entry gated on `deploy.swarm-otel.enable` — "does the swarm's collector run on THIS host". It had to be: loopback only reaches a reader that landed on the same host. What that rendered everywhere else was nothing at all. Off that host the metrics listener was not emitted, so the store's metrics reached the store nowhere, and a host with no entry is indistinguishable from a host nobody asked to scrape. Moves the scrape into the collector this container already runs, per mara on #4537: "move the existing scraper to the local collector". The container shares the host netns (privateNetwork = false), so the scrape still dials 127.0.0.1 — the listener keeps its address, its `metrics_only` narrowing and its loopback-only bind, and the API listener's `tls_require_and_verify_client_cert` is untouched. The listener and its `prometheus_retention_time` lose their gate: the reader ships with the store now, so there is no host where the endpoint has none. The metrics pipeline reuses the logs pipeline's `resource` processor and `otlphttp` exporter, so both signals carry the same `service.name` and leave by the one hop. Logs are unaffected: `journaldUnits` and --link-journal=host stay until every sibling swarm container has a collector of its own. The module-eval absence arm "a store with no collector beside it serves no metrics" is inverted rather than dropped — the condition it asserted is the bug. Three cases join it: the job is in swarm-bao AND gone from swarm-otel (a move, not a copy), the scrape target and listener are both pinned to loopback, and the metrics pipeline shares its exporter with the logs one.
313 lines
15 KiB
Nix
313 lines
15 KiB
Nix
# `checks.module-eval-secret-publisher` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
runGroup
|
|
;
|
|
|
|
# The IdP and the store on one machine: the shape where minted plaintext and
|
|
# a store identity are both present without an operator placing anything.
|
|
# Two hives in the roster, because the publisher walks it — an arm written
|
|
# against a single-hive fixture passes on a hardcoded name.
|
|
secretPublisherHere = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.authelia.enable = true;
|
|
swarm.hives.h2.domain = "h2.t.local";
|
|
};
|
|
|
|
# The IdP with no store on the box and a leaf placed by hand, which is the
|
|
# deployment this unit exists for: authelia is the one host the store is
|
|
# guaranteed not to share once either has a machine of its own.
|
|
#
|
|
# ⚠️ `enable` is deliberately NOT set here. It used to be, with a comment
|
|
# saying the default asked whether both ran on this host — which documented
|
|
# the co-location bug instead of catching it. Leaving it unset is what makes
|
|
# this fixture exercise the default rather than mask it.
|
|
secretPublisherRemote = hive {
|
|
deploy.authelia.enable = true;
|
|
deploy.swarm-secret-publisher.baoClientCertFile = "/etc/pki/publisher.pem";
|
|
deploy.swarm-secret-publisher.baoClientKeyFile = "/etc/pki/publisher-key.pem";
|
|
};
|
|
|
|
# The same IdP with the identity taken away. Minting the secrets is not being
|
|
# able to publish them, and this is the arm that separates the two.
|
|
secretPublisherNoIdentity = hive { deploy.authelia.enable = true; };
|
|
# Duplicated from grafana.nix — a case here needs it too.
|
|
|
|
# The same UI with the IdP on ANOTHER host and a store leaf placed by hand.
|
|
# Knowing an IdP is not running one: `swarm.authelia.url` is what says this
|
|
# swarm has SSO, and nothing about this host does. Identical to the fixture
|
|
# above in everything the delivery path reads, which is the point.
|
|
grafanaRemoteAuthelia = hive {
|
|
deploy.grafana.enable = true;
|
|
deploy.grafana.plugins = [ ];
|
|
deploy.grafana.package = pkgs.emptyDirectory;
|
|
swarm.authelia.url = "https://auth.example.invalid";
|
|
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
|
|
};
|
|
# Duplicated from swarm-otel-core.nix — a case here needs it too.
|
|
|
|
# The same collector with the IdP on ANOTHER host and a store leaf placed by
|
|
# hand. Identical to the fixture above in everything the delivery path
|
|
# reads, which is the point.
|
|
otelBaoRemoteAuthelia = hive {
|
|
deploy.swarm-otel.enable = true;
|
|
swarm.authelia.url = "https://auth.example.invalid";
|
|
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
|
|
};
|
|
# Duplicated from core-toggle.nix — a case here needs it too.
|
|
|
|
bare = hive { };
|
|
# Duplicated from bao-matrix-reader.nix — a case here needs it too.
|
|
|
|
# The store and a service that reads from it, versus the store alone. The
|
|
# pair is what makes the reader's absence arm mean anything.
|
|
baoWithMatrix = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.matrix.enable = true;
|
|
};
|
|
cases = [
|
|
{
|
|
# Both ends of a wire nothing at eval time carries end to end: the
|
|
# publisher on authelia's host writes the path the reader on Grafana's host
|
|
# reads, and the two files agree only because both compose it from the same
|
|
# swarm-wide client id.
|
|
name = "the publisher writes the swarm service path grafana reads";
|
|
ok = lib.hasInfix "secret/swarm/services/swarm-grafana/oidc/client" (
|
|
secretPublisherHere.systemd.services.swarm-secret-publish.script
|
|
);
|
|
}
|
|
{
|
|
# Registering the client cannot live where the rest of grafana's module
|
|
# lives: that block is gated on this host RUNNING grafana, so on the split
|
|
# deployment nothing registered the client, authelia minted no secret, and
|
|
# every layer below had nothing to carry. The second arm is the control —
|
|
# a host with no IdP registers nothing.
|
|
name = "the swarm's grafana client is registered wherever authelia runs";
|
|
ok =
|
|
let
|
|
clients = m: map (c: c.id) m.services.hyperhive.swarm.authelia.oidc.clients;
|
|
in
|
|
lib.elem "swarm-grafana" (clients secretPublisherHere)
|
|
&& !(lib.elem "swarm-grafana" (clients grafanaRemoteAuthelia));
|
|
}
|
|
{
|
|
# The collector's half of the same defect and the same fix: this used to
|
|
# be gated on `deploy.swarm-otel.enable`, so a split deployment
|
|
# registered the client nowhere and authelia minted nothing to publish.
|
|
name = "the swarm's collector client is registered wherever authelia runs";
|
|
ok =
|
|
let
|
|
clients = m: map (c: c.id) m.services.hyperhive.swarm.authelia.oidc.clients;
|
|
in
|
|
lib.elem "swarm-collector" (clients secretPublisherHere)
|
|
&& !(lib.elem "swarm-collector" (clients otelBaoRemoteAuthelia));
|
|
}
|
|
{
|
|
# Both ends of a wire nothing at eval time carries end to end: the
|
|
# publisher on authelia's host writes the path the reader on the
|
|
# collector's host reads, and the two files agree only because both
|
|
# compose it from the same swarm-wide client id. `secretPublisherHere`
|
|
# already grew this client when `serviceClientIds` did.
|
|
name = "the publisher writes the swarm service path the collector reads";
|
|
ok = lib.hasInfix "secret/swarm/services/swarm-collector/oidc/client" (
|
|
secretPublisherHere.systemd.services.swarm-secret-publish.script
|
|
);
|
|
}
|
|
{
|
|
# The third swarm service on that list, and the one whose absence is
|
|
# hardest to see: the store's own forwarder holds a client authelia
|
|
# registers and mints for, so every layer above looks complete while
|
|
# the one hop that CARRIES the value skips it and the reader on the
|
|
# store's host waits forever on a path nothing writes.
|
|
name = "the publisher carries the store forwarder's secret too";
|
|
ok = lib.hasInfix "secret/swarm/services/swarm-bao-collector/oidc/client" (
|
|
secretPublisherHere.systemd.services.swarm-secret-publish.script
|
|
);
|
|
}
|
|
{
|
|
# The same hole the controller's case above names, open a second time: the
|
|
# PKI script grew a third leaf and no case read it.
|
|
name = "the store mints a leaf for the secret publisher, and the publisher is pointed at it";
|
|
ok =
|
|
let
|
|
m = secretPublisherHere;
|
|
p = m.services.hyperhive.deploy.swarm-secret-publisher;
|
|
in
|
|
lib.hasInfix "secret-publisher.pem" m.systemd.services.swarm-bao-pki.script
|
|
&& p.baoClientCertFile == "/var/lib/swarm-bao-pki/secret-publisher.pem"
|
|
&& p.baoClientKeyFile == "/var/lib/swarm-bao-pki/secret-publisher-key.pem";
|
|
}
|
|
{
|
|
# mara caught this by reading, which means no arm existed for it: the
|
|
# default asked `authelia.enable && bao.enable`, so the split deployment
|
|
# this unit is FOR defaulted off and published nothing, silently.
|
|
#
|
|
# The second clause is the control. Without it this passes on a default
|
|
# of plain `true`, which would be a different bug with the same symptom
|
|
# — an IdP-less host claiming it publishes secrets it never mints.
|
|
name = "the publisher defaults on where secrets are minted, whether or not the store is local";
|
|
ok =
|
|
secretPublisherRemote.services.hyperhive.deploy.swarm-secret-publisher.enable
|
|
&& !bare.services.hyperhive.deploy.swarm-secret-publisher.enable;
|
|
}
|
|
{
|
|
# The one security property of this unit, and why its push cannot be
|
|
# rewritten into the obvious shape: `bao` is an external binary, so an
|
|
# argument is world-readable in /proc for the life of the call.
|
|
# `value=@<path>` hands it the path and bao opens the file itself.
|
|
#
|
|
# The second arm is what makes the first mean anything — `value=@` can
|
|
# sit one line above a command substitution that put the plaintext in
|
|
# argv anyway.
|
|
#
|
|
# ⚠️ Comments are stripped first, and that is not tidiness. A `script`
|
|
# renders its own comments into the text, and this unit's comments name
|
|
# the hazard verbatim so the next editor does not reintroduce it. Without
|
|
# the strip this case reads that warning and fails — a check the artifact
|
|
# defeats by DESCRIBING the thing it is checked for.
|
|
name = "the publisher hands bao the secret's path, never the secret";
|
|
ok =
|
|
let
|
|
s = secretPublisherHere.systemd.services.swarm-secret-publish.script;
|
|
code = lib.concatStringsSep "\n" (
|
|
lib.filter (l: builtins.match "[[:space:]]*#.*" l == null) (lib.splitString "\n" s)
|
|
);
|
|
in
|
|
lib.hasInfix "value=@" code && !(lib.hasInfix "$(cat" code);
|
|
}
|
|
{
|
|
# Two ends of a wire nothing at eval time carries end to end: this is the
|
|
# path `swarm_secret_client::queue` resolves for the reader. Both hives
|
|
# are asserted, so a publisher that knew one name rather than the roster
|
|
# fails here rather than on the second hive ever added to a swarm.
|
|
name = "the publisher writes every hive in the roster to that hive's own queue path";
|
|
ok =
|
|
let
|
|
s = secretPublisherHere.systemd.services.swarm-secret-publish.script;
|
|
in
|
|
lib.hasInfix "secret/swarm/hives/h1/queue/agent" s
|
|
&& lib.hasInfix "secret/swarm/hives/h2/queue/agent" s;
|
|
}
|
|
{
|
|
# The producer's end of the read `glue-matrix-bao-token.nix` already did.
|
|
# Both hives are asserted for the reason the queue case above gives: a
|
|
# publisher that knew one name rather than the roster would pass on a
|
|
# single-hive fixture and strand the second hive ever added — which is
|
|
# the two-hives-never-converge shape this slice exists to close.
|
|
name = "the publisher mints an appservice token for every hive and writes it to that hive's matrix path";
|
|
ok =
|
|
let
|
|
s = secretPublisherHere.systemd.services.swarm-secret-publish.script;
|
|
in
|
|
lib.hasInfix "secret/swarm/hives/h1/matrix/appservice-token" s
|
|
&& lib.hasInfix "secret/swarm/hives/h2/matrix/appservice-token" s
|
|
&& lib.hasInfix "/dev/urandom" s;
|
|
}
|
|
{
|
|
# What makes a re-publish idempotent. This principal is granted
|
|
# `create`/`update` and no `read`, so it cannot ask the store whether a
|
|
# hive already has a token — with nowhere to keep one, every run would
|
|
# mint a fresh value and rotate the swarm's token. A state directory is
|
|
# that somewhere, and nothing else in this unit needs one, so its absence
|
|
# means exactly this.
|
|
#
|
|
# The second arm is the mint's own guard: the state file is only written
|
|
# when it is missing or empty. Dropping that test leaves a unit that
|
|
# still has a state directory and still rotates on every boot.
|
|
name = "the publisher keeps the tokens it minted, and mints only when it holds none";
|
|
ok =
|
|
let
|
|
u = secretPublisherHere.systemd.services.swarm-secret-publish;
|
|
in
|
|
lib.hasInfix "matrix-appservice-token" (u.serviceConfig.StateDirectory or "")
|
|
&& u.serviceConfig.StateDirectoryMode or null == "0700"
|
|
&& lib.hasInfix "if [ ! -s \"$src\" ]" u.script;
|
|
}
|
|
{
|
|
# A property of the SET, not of one unit: both of these authenticate by
|
|
# certificate, and `BAO_CLIENT_CERT` is transport rather than identity, so
|
|
# a script that reaches `bao kv` without a token asks a token helper this
|
|
# host does not carry and fails before the store ever answers. `-token-only`
|
|
# is what keeps the token off the helper on the way back out.
|
|
#
|
|
# Ordering, not presence: the login has to come first, so the check is
|
|
# that nothing before it is a data command. Comments are stripped because
|
|
# both units explain this in prose directly above the code.
|
|
name = "the cert-identity bao units log in before their first read or write, and keep the token out of the helper";
|
|
ok =
|
|
let
|
|
code =
|
|
s:
|
|
lib.concatStringsSep "\n" (
|
|
lib.filter (l: builtins.match "[[:space:]]*#.*" l == null) (lib.splitString "\n" s)
|
|
);
|
|
holdsTokenFirst =
|
|
s:
|
|
let
|
|
c = code s;
|
|
in
|
|
lib.hasInfix "bao login" c
|
|
&& lib.hasInfix "-token-only" c
|
|
&& !(lib.hasInfix "bao kv" (lib.head (lib.splitString "bao login" c)));
|
|
in
|
|
holdsTokenFirst secretPublisherHere.systemd.services.swarm-secret-publish.script
|
|
&& holdsTokenFirst baoWithMatrix.systemd.services.swarm-bao-matrix-token.script
|
|
# Controls, so a clean verdict above means something. In order: a bare
|
|
# read is refused, a read placed before the login is refused, and a
|
|
# login that exists only in a comment is refused — that last one is the
|
|
# arm the comment-stripping exists for.
|
|
&& !(holdsTokenFirst "bao kv get -field=value secret/x")
|
|
&& !(holdsTokenFirst "bao kv get secret/x\nBAO_TOKEN=\"$(bao login -method=cert -token-only)\"")
|
|
&& !(holdsTokenFirst "# bao login -method=cert -token-only goes here\nbao kv get secret/x")
|
|
&& holdsTokenFirst "BAO_TOKEN=\"$(bao login -method=cert -token-only)\"\nbao kv get secret/x";
|
|
}
|
|
{
|
|
# The doctrine three glue files state, as a property a rewrite has to
|
|
# keep: a client is defined by holding a certificate the store accepts,
|
|
# never by standing next to the store. Gating this on `deploy.bao.enable`
|
|
# would have left the unit rendering only on the one deployment that has
|
|
# no use for it.
|
|
name = "a publisher holding an identity runs on a host with no store";
|
|
ok =
|
|
let
|
|
m = secretPublisherRemote;
|
|
in
|
|
!m.services.hyperhive.deploy.bao.enable
|
|
&& (m.systemd.services ? swarm-secret-publish)
|
|
&& (m.systemd.paths ? swarm-secret-publish);
|
|
}
|
|
{
|
|
# What makes the arm above able to fail. Minting the secrets is not being
|
|
# able to publish them: with no certificate the unit would fail a TLS
|
|
# handshake on every rotation, so it must not exist at all.
|
|
name = "an IdP host with no store identity renders no publisher";
|
|
ok =
|
|
let
|
|
m = secretPublisherNoIdentity;
|
|
in
|
|
m.services.hyperhive.deploy.authelia.enable
|
|
&& !(m.systemd.services ? swarm-secret-publish)
|
|
&& !(m.systemd.paths ? swarm-secret-publish);
|
|
}
|
|
];
|
|
in
|
|
runGroup "secret-publisher" cases
|