c10cc0e018
`swarm-bao-nats-callout-xkey` runs `ensure-callout-keys`, importing
/var/lib/swarm-nats-callout/{issuer,callout-user}.seed when present so the
keys keep their public halves, then fetches all three public halves from
the store. The verb removes each seed file once a read of the store returns
a seed; when the store's seed differs, it is kept and a warning names the
file. No tmpfiles rule touches those files, since one could run before the
import with the store sealed and leave the next run to mint a new key.
The server renders the issuer and user public halves from the fetched files,
checked for shape as the xkey's is, and does not start without them. The
responder reads both seeds from the store, so the copies into the container
and their LoadCredential entries are gone, and a tmpfiles rule removes the
copies a container still holds. `swarm-nats-callout-keys` is removed.
On the store's host the fetch unit is also ordered after
`swarm-bao-secret-publisher-policy`, so the publisher's denial of the seed
paths is written before the unit creates or keeps a seed there.
`deploy.nats.autoGenerateCallout`, `calloutUserPublicKey`,
`calloutIssuerPublicKey`, `calloutUserSeedFile` and `calloutIssuerSeedFile`
are removed with `mkRemovedOptionModule`, under `swarm.nats` too, each
message saying the keys live in the store.
362 lines
14 KiB
Nix
362 lines
14 KiB
Nix
# `checks.module-eval-nats-tls` — see ./lib.nix for the shared rationale (why
|
|
# this suite exists, naming convention, "evaluates not executes").
|
|
#
|
|
# The queue's name, its bao-issued leaf, and the clients that dial it.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
hiveWithController
|
|
runGroup
|
|
bridgePorts
|
|
;
|
|
|
|
natsName = "nats.t.local";
|
|
natsUrl = "tls://${natsName}:4222";
|
|
|
|
# Every service on one host. The queue, the store and every in-tree client of the queue
|
|
# are all here, so the scan below reads each of them.
|
|
allLocal = hiveWithController {
|
|
deploy.singleHostSwarm = true;
|
|
};
|
|
|
|
# The same host on the mesh.
|
|
allLocalMesh = hive {
|
|
deploy.singleHostSwarm = true;
|
|
deploy.wireguard.enable = true;
|
|
deploy.wireguard.address = "10.100.0.1/24";
|
|
};
|
|
|
|
# The queue on a host whose store is elsewhere: no local policy unit.
|
|
queueNoStore = hive {
|
|
deploy.nats.enable = true;
|
|
};
|
|
|
|
# The store's host with the hive's own reader pointed somewhere else.
|
|
natsOnStoreHostReaderMoved = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.nats.enable = true;
|
|
deploy.nats.autoGenerateCallout = true;
|
|
deploy.bao.clientCertFile = "/etc/pki/h1.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem";
|
|
};
|
|
|
|
# A queue host holding only the hive's own store reader, placed by hand: no
|
|
# store here, and no queue leaf named.
|
|
queueWithHiveReaderOnly = hive {
|
|
deploy.nats.enable = true;
|
|
deploy.nats.autoGenerateCallout = true;
|
|
deploy.bao.clientCertFile = "/etc/pki/h1.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem";
|
|
};
|
|
|
|
# A hive that is not the queue's host, with nothing about the queue's
|
|
# address set by hand: what every hive but one in a multi-host swarm looks
|
|
# like. The controller is on too, since it may run away from the queue.
|
|
#
|
|
# The forge's secret and the controller's store identity are what a hive
|
|
# away from authelia and the store already has to be handed, and none is an
|
|
# address; without them this hive would fail assertions that have nothing to
|
|
# do with the queue.
|
|
remoteSecrets = {
|
|
deploy.forgejo.sso.clientSecretFile = "/var/lib/forgejo-oidc/by-hand.secret";
|
|
deploy.swarm-controller.baoClientCertFile = "/var/lib/swarm-controller/bao-client.pem";
|
|
deploy.swarm-controller.baoClientKeyFile = "/var/lib/swarm-controller/bao-client-key.pem";
|
|
};
|
|
remote = hiveWithController (
|
|
lib.recursiveUpdate remoteSecrets { deploy.swarm-controller.enable = true; }
|
|
);
|
|
|
|
# The same hive given its status secret by hand, which is what turns
|
|
# publishing on away from the IdP's host.
|
|
remotePublishing = hiveWithController (
|
|
lib.recursiveUpdate remoteSecrets {
|
|
deploy.hive-controller.statusPublish.clientSecretFile = "/var/lib/secrets/hive-h1.secret";
|
|
}
|
|
);
|
|
|
|
# A real half-config: the secret, with the URL it would be presented at
|
|
# taken away.
|
|
remoteSecretNoUrl = hive (
|
|
lib.recursiveUpdate remoteSecrets {
|
|
deploy.hive-controller.statusPublish.clientSecretFile = "/var/lib/secrets/hive-h1.secret";
|
|
deploy.hive-controller.statusPublish.natsUrl = null;
|
|
}
|
|
);
|
|
|
|
failedAssertions = m: lib.filter (a: !a.assertion) m.assertions;
|
|
refusedStatusSecret =
|
|
m: lib.any (a: lib.hasInfix "status-publishing client secret" a.message) (failedAssertions m);
|
|
|
|
policyScript = allLocal.systemd.services.swarm-bao-nats-tls-policy.script;
|
|
leafUnit = allLocal.systemd.services.swarm-bao-nats-tls;
|
|
natsContainer = allLocal.containers.swarm-nats.config;
|
|
natsTls = natsContainer.services.nats.settings.tls;
|
|
|
|
# Every `(nats|tls)://…` in a string.
|
|
urlsIn =
|
|
s: map builtins.head (builtins.filter builtins.isList (builtins.split "((nats|tls)://[^ '\"]+)" s));
|
|
|
|
# Every in-tree queue client, found rather than listed. The Rust client reads
|
|
# its address from `<PREFIX>_NATS_URL` (`swarm_queue_client::QueueConfig::
|
|
# from_env`), so any unit on the host or in a container that is handed one
|
|
# carries a variable of that shape. The responder takes a flag instead.
|
|
# Keyed by where each came from, so the control below can name them.
|
|
clientUrls =
|
|
machine:
|
|
let
|
|
fromUnits =
|
|
where: services:
|
|
lib.concatLists (
|
|
lib.mapAttrsToList (
|
|
unit: s:
|
|
lib.mapAttrsToList (var: v: {
|
|
name = "${where}/${unit}/${var}";
|
|
value = v;
|
|
}) (lib.filterAttrs (var: v: lib.hasSuffix "_NATS_URL" var && v != null) (s.environment or { }))
|
|
) services
|
|
);
|
|
containerUnits = lib.concatLists (
|
|
lib.mapAttrsToList (c: cc: fromUnits c cc.config.systemd.services) machine.containers
|
|
);
|
|
# The responder runs beside the queue only, so a hive without one has
|
|
# none to read.
|
|
responder =
|
|
map
|
|
(u: {
|
|
name = "swarm-nats/swarm-nats-auth/--nats-url";
|
|
value = u;
|
|
})
|
|
(
|
|
lib.optionals (machine.containers ? swarm-nats) (
|
|
urlsIn machine.containers.swarm-nats.config.systemd.services.swarm-nats-auth.serviceConfig.ExecStart
|
|
)
|
|
);
|
|
in
|
|
lib.listToAttrs (fromUnits "host" machine.systemd.services ++ containerUnits ++ responder);
|
|
|
|
scanned = clientUrls allLocal;
|
|
|
|
cases = [
|
|
{
|
|
# Control first: a scan that found nothing would pass the next case
|
|
# vacuously. These are the four clients in the tree today.
|
|
name = "the client scan finds hive-c0re, the agents, the controller and the responder";
|
|
ok = lib.all (k: scanned ? ${k}) [
|
|
"host/hive-c0re/HIVE_C0RE_NATS_URL"
|
|
"host/hive-c0re/HIVE_AGENT_NATS_URL"
|
|
"host/swarm-controller/SWARM_CONTROLLER_NATS_URL"
|
|
"swarm-nats/swarm-nats-auth/--nats-url"
|
|
];
|
|
}
|
|
{
|
|
# The server requires TLS and its leaf carries the name alone, so a
|
|
# `nats://` URL or an address is a client that cannot connect. Every one
|
|
# found, not the four above: a client added later is held to it too.
|
|
name = "every in-tree queue client dials tls://<the queue's name>:4222";
|
|
ok = lib.all (u: u == natsUrl) (lib.attrValues scanned);
|
|
}
|
|
{
|
|
# The option defaults the scan reads through, so a client that stops
|
|
# reading them does not also escape the property above.
|
|
name = "the queue URL options default to the name on the queue's host";
|
|
ok =
|
|
let
|
|
d = allLocal.services.hyperhive.deploy;
|
|
in
|
|
d.hive-controller.statusPublish.natsUrl == natsUrl
|
|
&& d.hive-controller.queue.agentNatsUrl == natsUrl
|
|
&& allLocal.services.hyperhive.swarm.controller.queue.natsUrl == natsUrl;
|
|
}
|
|
{
|
|
name = "the queue's name is served by this host's resolver, at the bridge address";
|
|
ok =
|
|
lib.elem natsName allLocal.services.hyperhive.gateway.localNames
|
|
&& lib.elem "/${natsName}/${allLocal.services.hyperhive.network.bridgeIp}" allLocal.services.dnsmasq.settings.address;
|
|
}
|
|
{
|
|
name = "the queue's pki role issues for its name alone";
|
|
ok = lib.all (arg: lib.hasInfix arg policyScript) [
|
|
"roles/swarm-nats \\"
|
|
"allowed_domains=${lib.escapeShellArg natsName} \\"
|
|
"allow_bare_domains=true \\"
|
|
"allow_subdomains=false \\"
|
|
"allow_glob_domains=false \\"
|
|
"allow_localhost=false \\"
|
|
"allow_any_name=false \\"
|
|
"allow_ip_sans=false \\"
|
|
"server_flag=true \\"
|
|
"client_flag=false \\"
|
|
];
|
|
}
|
|
{
|
|
# Every `path` the policy names, not a search for the one expected: a
|
|
# second grant added later fails here.
|
|
name = "the queue's policy grants pki/issue/swarm-nats and nothing else";
|
|
ok =
|
|
let
|
|
paths = map builtins.head (
|
|
builtins.filter builtins.isList (builtins.split "path \"([^\"]*)\"" policyScript)
|
|
);
|
|
in
|
|
paths == [ "pki/issue/swarm-nats" ]
|
|
&& lib.hasInfix ''capabilities = ["update"]'' policyScript
|
|
&& lib.hasInfix "allowed_common_names=swarm-nats" policyScript
|
|
&& lib.hasInfix "token_policies=swarm-nats" policyScript;
|
|
}
|
|
{
|
|
# Mint to consume: the leaf the unit writes is the one the server reads,
|
|
# and the login leaf glue-bao-tls signs is the one the unit presents.
|
|
name = "the server serves the leaf the host unit issues, and the unit logs in as swarm-nats";
|
|
ok =
|
|
natsTls.cert_file == "/var/lib/swarm-nats-tls/cert.pem"
|
|
&& natsTls.key_file == "/run/credentials/nats.service/tls-key"
|
|
&& lib.elem "tls-key:/var/lib/swarm-nats-tls/key.pem" natsContainer.systemd.services.nats.serviceConfig.LoadCredential
|
|
&& allLocal.containers.swarm-nats.bindMounts ? "/var/lib/swarm-nats-tls"
|
|
&& lib.hasInfix "d=/var/lib/swarm-nats-tls" leafUnit.script
|
|
&& lib.hasInfix "pki/issue/swarm-nats" leafUnit.script
|
|
&& leafUnit.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/nats.pem"
|
|
&& lib.hasInfix "[ -s /var/lib/swarm-bao-pki/nats.pem ]" allLocal.systemd.services.swarm-bao-pki.script
|
|
&& lib.hasInfix "swarm-nats \"\" clientAuth" allLocal.systemd.services.swarm-bao-pki.script;
|
|
}
|
|
{
|
|
# The hive's own reader is not the queue's identity, and no path is
|
|
# derived from it: a queue host without the store that holds only the
|
|
# hive's reader gets the leaf unit that fails naming the options, like
|
|
# one holding nothing.
|
|
name = "a queue host without the store is refused when it holds only the hive's reader";
|
|
ok =
|
|
let
|
|
u = queueWithHiveReaderOnly.systemd.services.swarm-bao-nats-tls;
|
|
in
|
|
queueWithHiveReaderOnly.services.hyperhive.deploy.nats.baoClientCertFile == null
|
|
&& !(u.environment ? BAO_CLIENT_CERT)
|
|
&& lib.hasInfix "services.hyperhive.deploy.nats.baoClient{Cert,Key}File" u.script;
|
|
}
|
|
{
|
|
# The default is the file the minting unit writes, wherever the hive's
|
|
# own reader is pointed.
|
|
name = "on the store's host the queue's leaf is the one minted there, not one beside the hive's reader";
|
|
ok =
|
|
let
|
|
n = natsOnStoreHostReaderMoved.services.hyperhive.deploy.nats;
|
|
mint = natsOnStoreHostReaderMoved.systemd.services.swarm-bao-pki.script;
|
|
in
|
|
n.baoClientCertFile == "/var/lib/swarm-bao-pki/nats.pem"
|
|
&& n.baoClientKeyFile == "/var/lib/swarm-bao-pki/nats-key.pem"
|
|
&& lib.hasInfix "[ -s /var/lib/swarm-bao-pki/nats.pem ] ||" mint
|
|
&& lib.hasInfix "/var/lib/swarm-bao-pki nats " mint;
|
|
}
|
|
{
|
|
name = "the server requires TLS: no allow_non_tls";
|
|
ok = !(natsContainer.services.nats.settings ? allow_non_tls);
|
|
}
|
|
{
|
|
name = "4222 is open on wg-hive when this host is on the mesh, and never host-wide";
|
|
ok =
|
|
lib.elem 4222 allLocalMesh.networking.firewall.interfaces.wg-hive.allowedTCPPorts
|
|
&& !(lib.elem 4222 allLocalMesh.networking.firewall.allowedTCPPorts)
|
|
&& lib.elem 4222 (bridgePorts allLocalMesh);
|
|
}
|
|
{
|
|
name = "4222 is not opened on wg-hive when this host is not on the mesh";
|
|
ok =
|
|
!(lib.elem 4222
|
|
(allLocal.networking.firewall.interfaces.wg-hive or { allowedTCPPorts = [ ]; }).allowedTCPPorts
|
|
)
|
|
&& !(lib.elem 4222 allLocal.networking.firewall.allowedTCPPorts);
|
|
}
|
|
{
|
|
# Ordering, never a requirement: a policy unit that failed still counts
|
|
# as done, and the leaf unit's own retries carry it past that.
|
|
name = "the leaf unit is ordered after its policy unit, with no requires";
|
|
ok =
|
|
let
|
|
p = "swarm-bao-nats-tls-policy.service";
|
|
in
|
|
lib.elem p leafUnit.after && lib.elem p leafUnit.wants && !(lib.elem p (leafUnit.requires or [ ]));
|
|
}
|
|
{
|
|
name = "a queue host whose store is elsewhere orders its leaf unit after no policy unit";
|
|
ok =
|
|
let
|
|
u = queueNoStore.systemd.services.swarm-bao-nats-tls;
|
|
in
|
|
!(lib.elem "swarm-bao-nats-tls-policy.service" u.after)
|
|
&& !(lib.elem "swarm-bao-nats-tls-policy.service" u.wants);
|
|
}
|
|
{
|
|
# Every hive dials the queue by the same name, so a hive away from it
|
|
# needs no URL of its own. Control and property in one: the scan must
|
|
# reach the controller and the agents' address here, and every URL it
|
|
# finds, like the options it reads through, is the name.
|
|
name = "a hive that is not the queue's host dials tls://<the queue's name>:4222 with nothing set";
|
|
ok =
|
|
let
|
|
s = clientUrls remote;
|
|
d = remote.services.hyperhive.deploy;
|
|
in
|
|
!d.nats.enable
|
|
&& s ? "host/hive-c0re/HIVE_AGENT_NATS_URL"
|
|
&& s ? "host/swarm-controller/SWARM_CONTROLLER_NATS_URL"
|
|
&& lib.all (u: u == natsUrl) (lib.attrValues s)
|
|
&& d.hive-controller.statusPublish.natsUrl == natsUrl
|
|
&& d.hive-controller.queue.agentNatsUrl == natsUrl
|
|
&& remote.services.hyperhive.swarm.controller.queue.natsUrl == natsUrl;
|
|
}
|
|
{
|
|
# The URL is set and the secret is not, which is every such hive until
|
|
# an operator places one: publishing is off, not misconfigured.
|
|
name = "that hive evaluates without an assertion failure";
|
|
ok = failedAssertions remote == [ ];
|
|
}
|
|
{
|
|
# Off means off: hive-c0re is handed no status coordinates, rather
|
|
# than a secret path nothing fills.
|
|
name = "without its status secret, that hive's hive-c0re is given no status coordinates";
|
|
ok =
|
|
let
|
|
s = remote.systemd.services.hive-c0re;
|
|
in
|
|
!(s.environment ? HIVE_C0RE_NATS_URL)
|
|
&& !(s.environment ? HIVE_C0RE_OIDC_CLIENT_SECRET_FILE)
|
|
&& !(lib.any (lib.hasPrefix "swarm-status-client.secret:") (
|
|
lib.toList (s.serviceConfig.LoadCredential or [ ])
|
|
));
|
|
}
|
|
{
|
|
# The secret alone turns publishing on, at the default URL.
|
|
name = "given its status secret, that hive publishes to the queue's name";
|
|
ok =
|
|
let
|
|
s = remotePublishing.systemd.services.hive-c0re;
|
|
in
|
|
failedAssertions remotePublishing == [ ]
|
|
&& s.environment.HIVE_C0RE_NATS_URL == natsUrl
|
|
&& s.environment.HIVE_C0RE_OIDC_CLIENT_SECRET_FILE == "%d/swarm-status-client.secret"
|
|
&& lib.elem "swarm-status-client.secret:/var/lib/secrets/hive-h1.secret" (
|
|
lib.toList s.serviceConfig.LoadCredential
|
|
);
|
|
}
|
|
{
|
|
# What the assertion was written for is still refused: a secret with
|
|
# nowhere to present it.
|
|
name = "a status secret without a queue URL is refused at eval";
|
|
ok = refusedStatusSecret remoteSecretNoUrl && !(refusedStatusSecret remote);
|
|
}
|
|
];
|
|
in
|
|
runGroup "nats-tls" cases
|