vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-nats: take the callout issuer and user keys from the store

Browse source 
`swarm-bao-nats-callout-xkey` runs `ensure-callout-keys`, importing
/var/lib/swarm-nats-callout/{issuer,callout-user}.seed when present so the
keys keep their public halves, then fetches all three public halves from
the store. The verb removes each seed file once a read of the store returns
a seed; when the store's seed differs, it is kept and a warning names the
file. No tmpfiles rule touches those files, since one could run before the
import with the store sealed and leave the next run to mint a new key.

The server renders the issuer and user public halves from the fetched files,
checked for shape as the xkey's is, and does not start without them. The
responder reads both seeds from the store, so the copies into the container
and their LoadCredential entries are gone, and a tmpfiles rule removes the
copies a container still holds. `swarm-nats-callout-keys` is removed.

On the store's host the fetch unit is also ordered after
`swarm-bao-secret-publisher-policy`, so the publisher's denial of the seed
paths is written before the unit creates or keeps a seed there.

`deploy.nats.autoGenerateCallout`, `calloutUserPublicKey`,
`calloutIssuerPublicKey`, `calloutUserSeedFile` and `calloutIssuerSeedFile`
are removed with `mkRemovedOptionModule`, under `swarm.nats` too, each
message saying the keys live in the store.
This commit is contained in:
atlas 2026-10-06 11:46:28 +02:00 • committed by mara
commit c10cc0e018
12 changed files with 427 additions and 556 deletions
Download patch file Download diff file Expand all files Collapse all files

156
nix/module-eval/nats-authelia.nix
View file

@ -21,20 +21,11 @@ let
runGroup
;
# The queue's callout identity, fourth split slice. `autoGenerateCallout` is
# left FALSE on purpose: that is what makes the seed paths the thing deciding
# `responderConfigured`, so the assertion below is about the seeds rather
# than about the auto-mint branch. Every one of the seven old paths is
# defined — `enable` included, which is why it is spelled the old way here
# while the fixture below uses the new one — so dropping any single nats
# shim fails the eval, not just the arms read.
# The queue's two old `swarm.nats` paths that still rename, `enable` and
# `authPackage`, so dropping either shim fails the eval. No store identity,
# which the first case below relies on.
natsOldPath = hive {
swarm.nats.enable = true;
swarm.nats.autoGenerateCallout = false;
swarm.nats.calloutUserPublicKey = "UTESTUSERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
swarm.nats.calloutIssuerPublicKey = "ATESTISSUERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
swarm.nats.calloutUserSeedFile = "/run/secrets/nats-user.seed";
swarm.nats.calloutIssuerSeedFile = "/run/secrets/nats-issuer.seed";
swarm.nats.authPackage = pkgs.emptyDirectory;
};
@ -53,22 +44,12 @@ let
swarm.authelia.usersFile = "/var/lib/test-authelia/users.yml";
swarm.authelia.package = pkgs.emptyDirectory;
swarm.authelia.bridgePackage = pkgs.emptyDirectory;
swarm.nats.autoGenerateCallout = false;
swarm.nats.calloutUserPublicKey = "UTESTUSERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
swarm.nats.calloutIssuerPublicKey = "ATESTISSUERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
swarm.nats.calloutUserSeedFile = "/run/secrets/nats-user.seed";
swarm.nats.calloutIssuerSeedFile = "/run/secrets/nats-issuer.seed";
};
# The queue with a store identity for its responder, placed by hand: the
# queue host that is not the store's.
natsWithStoreIdentity = hive {
deploy.nats.enable = true;
deploy.nats.autoGenerateCallout = false;
deploy.nats.calloutUserPublicKey = "UTESTUSERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
deploy.nats.calloutIssuerPublicKey = "ATESTISSUERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
deploy.nats.calloutUserSeedFile = "/run/secrets/nats-user.seed";
deploy.nats.calloutIssuerSeedFile = "/run/secrets/nats-issuer.seed";
deploy.nats.authPackage = pkgs.emptyDirectory;
deploy.nats.authBaoClientCertFile = "/etc/pki/nats-auth.pem";
deploy.nats.authBaoClientKeyFile = "/etc/pki/nats-auth-key.pem";
@ -92,11 +73,6 @@ let
# store here, and no responder leaf named.
natsWithHiveReaderOnly = hive {
deploy.nats.enable = true;
deploy.nats.autoGenerateCallout = false;
deploy.nats.calloutUserPublicKey = "UTESTUSERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
deploy.nats.calloutIssuerPublicKey = "ATESTISSUERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
deploy.nats.calloutUserSeedFile = "/run/secrets/nats-user.seed";
deploy.nats.calloutIssuerSeedFile = "/run/secrets/nats-issuer.seed";
deploy.nats.authPackage = pkgs.emptyDirectory;
deploy.bao.clientCertFile = "/etc/pki/h1.pem";
deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem";
@ -205,42 +181,80 @@ let
ok = lib.elem "nats.service" (responderOf natsOldPath).wantedBy;
}
{
# The key is created before anything needs it, by the responder's own
# verb under its own leaf, then read back from the public path and left
# where the server's render reads it. Before the container and wanted by
# it, and a fetch that finds nothing fails so `storeRetry` retries.
name = "the callout xkey is created, then fetched from the store before the queue's container starts";
# The keys are created before anything needs them, by the responder's
# own verb under its own leaf, then each public half is read back from
# the store and left where the server's render reads it. Before the
# container and wanted by it, and a fetch that finds nothing fails so
# `storeRetry` retries.
name = "the callout keys are created, then their public halves fetched from the store before the queue's container starts";
ok =
let
u = natsWithStoreIdentity.systemd.services.swarm-bao-nats-callout-xkey;
s = builtins.unsafeDiscardStringContext u.script;
at = needle: lib.stringLength (lib.head (lib.splitString needle s));
ensure = "swarm-nats-auth ensure-xkey --store-cert-role swarm-nats-auth";
fetch = "bao kv get -field=value secret/swarm/public/swarm-nats/callout-xkey";
file = "/var/lib/nixos-containers/swarm-nats/var/lib/nats-callout/callout-xkey.pub";
ensure = "swarm-nats-auth ensure-callout-keys --store-cert-role swarm-nats-auth";
fetched =
leaf: file:
let
fetch = "bao kv get -field=value secret/swarm/public/swarm-nats/${leaf}";
path = "/var/lib/nixos-containers/swarm-nats/var/lib/nats-callout/${file}";
in
lib.hasInfix fetch s
&& at ensure < at fetch
&& lib.hasInfix "atomic_write_secret 0444 \"\" ${path}" s
&& lib.hasInfix "refresh_consumer swarm-nats nats.service ${path}" s;
in
lib.elem "container@swarm-nats.service" u.before
&& lib.elem "container@swarm-nats.service" u.wantedBy
&& u.environment.BAO_CLIENT_CERT == "/etc/pki/nats-auth.pem"
&& lib.hasInfix ensure s
&& lib.hasInfix fetch s
&& at ensure < at fetch
&& lib.hasInfix "atomic_write_secret 0444 \"\" ${file}" s
&& lib.hasInfix "refresh_consumer swarm-nats nats.service ${file}" s
&& fetched "callout-xkey" "callout-xkey.pub"
&& fetched "callout-issuer" "issuer.pub"
&& fetched "callout-user" "callout-user.pub"
&& !(lib.hasInfix "exit 0" s)
&& u.serviceConfig.Restart == "on-failure";
}
{
# Its creates are what the responder's role was granted; on the store's
# host the role's unit has to have run first.
name = "on the store's host the callout xkey fetch waits for the responder's role";
# The seeds a host kept on disk are what the verb stores when the store
# holds none, so the keys keep their public halves; the verb removes
# them itself. Nothing mints a seed on the host, and no
# tmpfiles rule touches the host seeds, which could run before the
# import does.
name = "the callout keys import the host's seed files, which only the importing unit removes";
ok =
let
s = natsWithStoreIdentity.systemd.services.swarm-bao-nats-callout-xkey.script;
rules = natsWithStoreIdentity.systemd.tmpfiles.rules;
in
lib.hasInfix "--import-issuer-seed-file /var/lib/swarm-nats-callout/issuer.seed" s
&& lib.hasInfix "--import-user-seed-file /var/lib/swarm-nats-callout/callout-user.seed" s
&& !(natsWithStoreIdentity.systemd.services ? swarm-nats-callout-keys)
&& !(lib.any (lib.hasInfix "/var/lib/swarm-nats-callout") rules);
}
{
# Its creates are what the responder's role was granted, and the
# publisher's policy is what denies the publisher the seeds' paths: a
# seed found there is kept. On the store's host both units run first.
name = "on the store's host the callout key fetch waits for the responder's role and the publisher's denial";
ok =
let
u = natsOnStoreHost.systemd.services.swarm-bao-nats-callout-xkey;
waitsFor = p: lib.elem "${p}.service" u.after && lib.elem "${p}.service" u.wants;
in
u.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/nats-auth.pem"
&& lib.elem "swarm-bao-nats-auth-policy.service" u.after
&& lib.elem "swarm-bao-nats-auth-policy.service" u.wants;
&& waitsFor "swarm-bao-nats-auth-policy"
&& waitsFor "swarm-bao-secret-publisher-policy"
&& natsOnStoreHost.systemd.services ? swarm-bao-secret-publisher-policy;
}
{
# The control is the fixture above the store: off the store's host
# there is no local policy unit to wait for.
name = "off the store's host the callout key fetch waits for no policy unit";
ok =
let
u = natsWithStoreIdentity.systemd.services.swarm-bao-nats-callout-xkey;
in
!(lib.any (lib.hasSuffix "-policy.service") (u.after ++ u.wants));
}
{
# The responder refuses the hive's shared agent client and takes no flag
@ -318,9 +332,10 @@ let
&& lib.hasPrefix "https://" u.environment.BAO_ADDR;
}
{
# Copied as the seeds are, so an absent leaf fails the unit rather than
# reaching the responder as a file it cannot log in with.
name = "the copy unit delivers the store identity beside the two seeds, and nothing in its place when absent";
# Copied so an absent leaf fails the unit rather than reaching the
# responder as a file it cannot log in with. No seed is copied: the
# responder reads every callout seed from the store.
name = "the copy unit delivers the store identity and no seed, and nothing in its place when absent";
ok =
let
s = natsWithStoreIdentity.systemd.services.swarm-nats-auth-secrets.script;
@ -328,8 +343,51 @@ let
lib.hasInfix "install -m 0400 /etc/pki/nats-auth.pem " s
&& lib.hasInfix "install -m 0400 /etc/pki/nats-auth-key.pem " s
&& !(lib.hasInfix "/dev/null" s)
&& lib.hasInfix "callout-user.seed" s
&& lib.hasInfix "issuer.seed" s;
&& !(lib.hasInfix ".seed" s);
}
{
# The responder names no seed file, by flag or by credential, and the
# copies a container may still hold are removed. The OIDC secret is the
# control that the credential list is read at all.
name = "the responder takes no seed from a file, and the container's seed copies are removed";
ok =
let
u = responderOf natsWithStoreIdentity;
creds = u.serviceConfig.LoadCredential;
dir = "/var/lib/nixos-containers/swarm-nats/var/lib/swarm-nats-auth";
rules = natsWithStoreIdentity.systemd.tmpfiles.rules;
in
lib.elem "oidc-client.secret:/var/lib/swarm-nats-auth/oidc-client.secret" creds
&& !(lib.any (lib.hasInfix ".seed") creds)
&& !(lib.hasInfix "seed-file" u.serviceConfig.ExecStart)
&& lib.elem "r ${dir}/issuer.seed" rules
&& lib.elem "r ${dir}/callout-user.seed" rules;
}
{
# Under either namespace, each removed option fails evaluation naming
# itself and the store. The control is a queue host that sets none.
name = "a definition of a removed callout option fails evaluation, naming the option and the secret store";
ok =
let
refused =
m: path:
lib.any (
a:
!a.assertion
&& lib.hasInfix "services.hyperhive.${path}" a.message
&& lib.hasInfix "live in the swarm secret store" a.message
) m.assertions;
setting = {
"deploy.nats.autoGenerateCallout" = hive { deploy.nats.autoGenerateCallout = true; };
"swarm.nats.autoGenerateCallout" = hive { swarm.nats.autoGenerateCallout = true; };
"deploy.nats.calloutUserPublicKey" = hive { deploy.nats.calloutUserPublicKey = "U"; };
"deploy.nats.calloutIssuerPublicKey" = hive { deploy.nats.calloutIssuerPublicKey = "A"; };
"deploy.nats.calloutUserSeedFile" = hive { deploy.nats.calloutUserSeedFile = "/run/u.seed"; };
"swarm.nats.calloutIssuerSeedFile" = hive { swarm.nats.calloutIssuerSeedFile = "/run/i.seed"; };
};
in
lib.all (path: refused setting.${path} path) (lib.attrNames setting)
&& !(lib.any (path: refused natsWithStoreIdentity path) (lib.attrNames setting));
}
{
# The responder's OIDC secret has one route, the store, under the