vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

hive status publishing: read the hive's OIDC secret from the store, retry until it is there

Browse source 
hive-c0re publishes this hive's status to the swarm queue as the OIDC
client `hive-<name>`. Its secret came from
`deploy.hive-controller.statusPublish.clientSecretFile`, which defaulted
to authelia's minted file only on a host running both authelia and the
queue. Every other hive had no secret and never published, so the
controller showed it `never_reported`.

Now `swarm-secret-publish`, beside authelia, also copies each hive's
`hive-<name>` secret to `swarm/hives/<hive>/oidc/client`. Its policy
already allows `swarm/hives/*`. hive-c0re reads that path under the
hive's own store leaf, which the existing `swarm/hives/<hive>/*` grant
covers, and holds the value in memory.

The shared queue client no longer remembers a failed first connect for
the life of the process. A failed attempt is retried with a backoff
from 5s, doubling to a cap of 5 minutes; the dashboard banner stays up
until an attempt succeeds. The two status publishers wait for the
client instead of giving up on the first `None`, so a secret published
after boot is picked up without a restart.

Removed: the `statusPublish.clientSecretFile` option (both spellings,
now removed options with a message), its same-host default, its
LoadCredential, the assertion tying it to the other two coordinates,
and `swarm_queue_client::QueueConfig::from_env`, whose only caller was
this read. Publishing is now on wherever the queue URL and token
endpoint are set and the hive holds a store leaf.

Refs #5015
This commit is contained in:
atlas 2026-10-09 23:12:28 +02:00 • committed by mara
commit c2c61b6146
19 changed files with 643 additions and 394 deletions
Download patch file Download diff file Expand all files Collapse all files

86
nix/module-eval/nats-tls.nix
View file

@ -80,26 +80,26 @@ let
lib.recursiveUpdate remoteSecrets { deploy.swarm-controller.enable = true; }
);
# The same hive given its status secret by hand, which is what turns
# publishing on away from the IdP's host.
remotePublishing = hiveWithController (
lib.recursiveUpdate remoteSecrets {
deploy.hive-controller.statusPublish.clientSecretFile = "/var/lib/secrets/hive-h1.secret";
}
);
# The same hive given its own store leaf by hand, which is what turns
# publishing on away from the IdP's host: hive-c0re reads the status secret
# from the store under it. The queue-credential reader's pair comes with it,
# because a hive that reads the store is refused at eval without one.
hiveLeaf = {
deploy.bao.clientCertFile = "/etc/pki/h1.pem";
deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem";
deploy.bao.queueAgentClientCertFile = "/etc/pki/h1-queue-agent.pem";
deploy.bao.queueAgentClientKeyFile = "/etc/pki/h1-queue-agent-key.pem";
};
remotePublishing = hiveWithController (lib.recursiveUpdate remoteSecrets hiveLeaf);
# A real half-config: the secret, with the URL it would be presented at
# taken away.
remoteSecretNoUrl = hive (
lib.recursiveUpdate remoteSecrets {
deploy.hive-controller.statusPublish.clientSecretFile = "/var/lib/secrets/hive-h1.secret";
deploy.hive-controller.statusPublish.natsUrl = null;
}
# The same leaf with the queue's URL taken away.
remoteLeafNoUrl = hive (
lib.recursiveUpdate remoteSecrets (
lib.recursiveUpdate hiveLeaf { deploy.hive-controller.statusPublish.natsUrl = null; }
)
);
failedAssertions = m: lib.filter (a: !a.assertion) m.assertions;
refusedStatusSecret =
m: lib.any (a: lib.hasInfix "status-publishing client secret" a.message) (failedAssertions m);
policyScript = allLocal.systemd.services.swarm-bao-nats-tls-policy.script;
leafUnit = allLocal.systemd.services.swarm-bao-nats-tls;
@ -318,44 +318,50 @@ let
&& remote.services.hyperhive.swarm.controller.queue.natsUrl == natsUrl;
}
{
# The URL is set and the secret is not, which is every such hive until
# an operator places one: publishing is off, not misconfigured.
# The URL is set and the hive holds no store leaf, which is every such
# hive until an operator places one: publishing is off, not
# misconfigured.
name = "that hive evaluates without an assertion failure";
ok = failedAssertions remote == [ ];
}
{
# Off means off: hive-c0re is handed no status coordinates, rather
# than a secret path nothing fills.
name = "without its status secret, that hive's hive-c0re is given no status coordinates";
ok =
let
s = remote.systemd.services.hive-c0re;
in
!(s.environment ? HIVE_C0RE_NATS_URL)
&& !(s.environment ? HIVE_C0RE_OIDC_CLIENT_SECRET_FILE)
&& !(lib.any (lib.hasPrefix "swarm-status-client.secret:") (
lib.toList (s.serviceConfig.LoadCredential or [ ])
));
# Off means off: hive-c0re is handed no status coordinates it could not
# fetch a secret for.
name = "without a store leaf, that hive's hive-c0re is given no status coordinates";
ok = !(remote.systemd.services.hive-c0re.environment ? HIVE_C0RE_NATS_URL);
}
{
# The secret alone turns publishing on, at the default URL.
name = "given its status secret, that hive publishes to the queue's name";
# The leaf alone turns publishing on, at the default URL, with the
# secret read from the store rather than named as a file.
name = "given its store leaf, that hive publishes to the queue's name as hive-<name>";
ok =
let
s = remotePublishing.systemd.services.hive-c0re;
in
failedAssertions remotePublishing == [ ]
&& s.environment.HIVE_C0RE_NATS_URL == natsUrl
&& s.environment.HIVE_C0RE_OIDC_CLIENT_SECRET_FILE == "%d/swarm-status-client.secret"
&& lib.elem "swarm-status-client.secret:/var/lib/secrets/hive-h1.secret" (
lib.toList s.serviceConfig.LoadCredential
);
&& s.environment.HIVE_C0RE_OIDC_CLIENT_ID == "hive-h1"
&& !(s.environment ? HIVE_C0RE_OIDC_CLIENT_SECRET_FILE);
}
{
# What the assertion was written for is still refused: a secret with
# nowhere to present it.
name = "a status secret without a queue URL is refused at eval";
ok = refusedStatusSecret remoteSecretNoUrl && !(refusedStatusSecret remote);
# The host that runs the queue, the IdP and the store gets the same
# coordinates from the leaf ../host-modules/glue-bao-tls.nix mints there,
# and no file.
name = "the all-local hive publishes from the store too";
ok =
let
e = allLocal.systemd.services.hive-c0re.environment;
in
e.HIVE_C0RE_NATS_URL == natsUrl
&& e.HIVE_C0RE_OIDC_CLIENT_ID == "hive-h1"
&& !(e ? HIVE_C0RE_OIDC_CLIENT_SECRET_FILE);
}
{
# A null URL turns publishing off.
name = "a hive with a store leaf and no queue URL is given no status coordinates";
ok =
failedAssertions remoteLeafNoUrl == [ ]
&& !(remoteLeafNoUrl.systemd.services.hive-c0re.environment ? HIVE_C0RE_NATS_URL);
}
];
in

18
nix/module-eval/secret-publisher.nix
View file

@ -84,6 +84,24 @@ let
deploy.matrix.enable = true;
};
cases = [
{
# Each hive's hive-c0re reads this exact path
# (`swarm_secret_client::queue::hive_client_path`, pinned by that crate's
# own test), for every hive in the roster, not only the one this host is.
# The agents' client beside it stays a separate path.
name = "the publisher writes every hive's own client under that hive's prefix";
ok =
let
s = secretPublisherHere.systemd.services.swarm-secret-publish.script;
in
lib.all
(h: lib.hasInfix "secret/swarm/hives/${h}/oidc/client" s && lib.hasInfix "/hive-${h}.secret" s)
[
"h1"
"h2"
]
&& lib.hasInfix "secret/swarm/hives/h1/queue/agent" s;
}
{
# The controller reads this exact path
# (`swarm_secret_client::queue::controller_client_path`, pinned by that