a5259146dc
A remote hive dialled nothing until an operator copied the queue's URL into it, though the URL is the same string everywhere. statusPublish.natsUrl, queue.agentNatsUrl and controller.queue.natsUrl now default to tls://<swarm.nats.domain>:<port> unconditionally. The statusPublish assertion treated a URL without a secret as a half config. With the URL a default on every hive, only the secret claims publishing: the assertion now refuses a secret without a URL or token endpoint, and hive-c0re's status environment is gated on the secret too, so a hive without one publishes nothing instead of reading a missing credential.
314 lines
12 KiB
Nix
314 lines
12 KiB
Nix
# `checks.module-eval-nats-tls` — see ./lib.nix for the shared rationale (why
|
|
# this suite exists, naming convention, "evaluates not executes").
|
|
#
|
|
# The queue's name, its bao-issued leaf, and the clients that dial it.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
runGroup
|
|
bridgePorts
|
|
;
|
|
|
|
natsName = "nats.t.local";
|
|
natsUrl = "tls://${natsName}:4222";
|
|
|
|
# Every service on one host, with a bootstrap token so the store's granting
|
|
# units render. The queue, the store and every in-tree client of the queue
|
|
# are all here, so the scan below reads each of them.
|
|
allLocal = hive {
|
|
deploy.singleHostSwarm = true;
|
|
deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token";
|
|
};
|
|
|
|
# The same host on the mesh.
|
|
allLocalMesh = hive {
|
|
deploy.singleHostSwarm = true;
|
|
deploy.wireguard.enable = true;
|
|
deploy.wireguard.address = "10.100.0.1/24";
|
|
};
|
|
|
|
# The queue on a host whose store is elsewhere: no local policy unit.
|
|
queueNoStore = hive {
|
|
deploy.nats.enable = true;
|
|
deploy.nats.autoGenerateCallout = true;
|
|
};
|
|
|
|
# A hive that is not the queue's host, with nothing about the queue's
|
|
# address set by hand: what every hive but one in a multi-host swarm looks
|
|
# like. The controller is on too, since it may run away from the queue.
|
|
#
|
|
# The two secrets are the ones a hive away from authelia already has to be
|
|
# handed, and neither is an address; without them this hive would fail
|
|
# assertions that have nothing to do with the queue.
|
|
remoteSecrets = {
|
|
deploy.forgejo.sso.clientSecretFile = "/var/lib/forgejo-oidc/by-hand.secret";
|
|
deploy.swarm-controller.queue.clientSecretFile = "/var/lib/secrets/swarm-controller.secret";
|
|
};
|
|
remote = hive (lib.recursiveUpdate remoteSecrets { deploy.swarm-controller.enable = true; });
|
|
|
|
# The same hive given its status secret by hand, which is what turns
|
|
# publishing on away from the IdP's host.
|
|
remotePublishing = hive (
|
|
lib.recursiveUpdate remoteSecrets {
|
|
deploy.hive-controller.statusPublish.clientSecretFile = "/var/lib/secrets/hive-h1.secret";
|
|
}
|
|
);
|
|
|
|
# A real half-config: the secret, with the URL it would be presented at
|
|
# taken away.
|
|
remoteSecretNoUrl = hive (
|
|
lib.recursiveUpdate remoteSecrets {
|
|
deploy.hive-controller.statusPublish.clientSecretFile = "/var/lib/secrets/hive-h1.secret";
|
|
deploy.hive-controller.statusPublish.natsUrl = null;
|
|
}
|
|
);
|
|
|
|
failedAssertions = m: lib.filter (a: !a.assertion) m.assertions;
|
|
refusedStatusSecret =
|
|
m: lib.any (a: lib.hasInfix "status-publishing client secret" a.message) (failedAssertions m);
|
|
|
|
policyScript = allLocal.systemd.services.swarm-bao-nats-tls-policy.script;
|
|
leafUnit = allLocal.systemd.services.swarm-bao-nats-tls;
|
|
natsContainer = allLocal.containers.swarm-nats.config;
|
|
natsTls = natsContainer.services.nats.settings.tls;
|
|
|
|
# Every `(nats|tls)://…` in a string.
|
|
urlsIn =
|
|
s: map builtins.head (builtins.filter builtins.isList (builtins.split "((nats|tls)://[^ '\"]+)" s));
|
|
|
|
# Every in-tree queue client, found rather than listed. The Rust client reads
|
|
# its address from `<PREFIX>_NATS_URL` (`swarm_queue_client::QueueConfig::
|
|
# from_env`), so any unit on the host or in a container that is handed one
|
|
# carries a variable of that shape. The responder takes a flag instead.
|
|
# Keyed by where each came from, so the control below can name them.
|
|
clientUrls =
|
|
machine:
|
|
let
|
|
fromUnits =
|
|
where: services:
|
|
lib.concatLists (
|
|
lib.mapAttrsToList (
|
|
unit: s:
|
|
lib.mapAttrsToList (var: v: {
|
|
name = "${where}/${unit}/${var}";
|
|
value = v;
|
|
}) (lib.filterAttrs (var: v: lib.hasSuffix "_NATS_URL" var && v != null) (s.environment or { }))
|
|
) services
|
|
);
|
|
containerUnits = lib.concatLists (
|
|
lib.mapAttrsToList (c: cc: fromUnits c cc.config.systemd.services) machine.containers
|
|
);
|
|
# The responder runs beside the queue only, so a hive without one has
|
|
# none to read.
|
|
responder =
|
|
map
|
|
(u: {
|
|
name = "swarm-nats/swarm-nats-auth/--nats-url";
|
|
value = u;
|
|
})
|
|
(
|
|
lib.optionals (machine.containers ? swarm-nats) (
|
|
urlsIn machine.containers.swarm-nats.config.systemd.services.swarm-nats-auth.serviceConfig.ExecStart
|
|
)
|
|
);
|
|
in
|
|
lib.listToAttrs (fromUnits "host" machine.systemd.services ++ containerUnits ++ responder);
|
|
|
|
scanned = clientUrls allLocal;
|
|
|
|
cases = [
|
|
{
|
|
# Control first: a scan that found nothing would pass the next case
|
|
# vacuously. These are the four clients in the tree today.
|
|
name = "the client scan finds hive-c0re, the agents, the controller and the responder";
|
|
ok = lib.all (k: scanned ? ${k}) [
|
|
"host/hive-c0re/HIVE_C0RE_NATS_URL"
|
|
"host/hive-c0re/HIVE_AGENT_NATS_URL"
|
|
"host/swarm-controller/SWARM_CONTROLLER_NATS_URL"
|
|
"swarm-nats/swarm-nats-auth/--nats-url"
|
|
];
|
|
}
|
|
{
|
|
# The server requires TLS and its leaf carries the name alone, so a
|
|
# `nats://` URL or an address is a client that cannot connect. Every one
|
|
# found, not the four above: a client added later is held to it too.
|
|
name = "every in-tree queue client dials tls://<the queue's name>:4222";
|
|
ok = lib.all (u: u == natsUrl) (lib.attrValues scanned);
|
|
}
|
|
{
|
|
# The option defaults the scan reads through, so a client that stops
|
|
# reading them does not also escape the property above.
|
|
name = "the queue URL options default to the name on the queue's host";
|
|
ok =
|
|
let
|
|
d = allLocal.services.hyperhive.deploy;
|
|
in
|
|
d.hive-controller.statusPublish.natsUrl == natsUrl
|
|
&& d.hive-controller.queue.agentNatsUrl == natsUrl
|
|
&& allLocal.services.hyperhive.swarm.controller.queue.natsUrl == natsUrl;
|
|
}
|
|
{
|
|
name = "the queue's name is served by this host's resolver, at the bridge address";
|
|
ok =
|
|
lib.elem natsName allLocal.services.hyperhive.gateway.localNames
|
|
&& lib.elem "/${natsName}/${allLocal.services.hyperhive.network.bridgeIp}" allLocal.services.dnsmasq.settings.address;
|
|
}
|
|
{
|
|
name = "the queue's pki role issues for its name alone";
|
|
ok = lib.all (arg: lib.hasInfix arg policyScript) [
|
|
"roles/swarm-nats \\"
|
|
"allowed_domains=${lib.escapeShellArg natsName} \\"
|
|
"allow_bare_domains=true \\"
|
|
"allow_subdomains=false \\"
|
|
"allow_glob_domains=false \\"
|
|
"allow_localhost=false \\"
|
|
"allow_any_name=false \\"
|
|
"allow_ip_sans=false \\"
|
|
"server_flag=true \\"
|
|
"client_flag=false \\"
|
|
];
|
|
}
|
|
{
|
|
# Every `path` the policy names, not a search for the one expected: a
|
|
# second grant added later fails here.
|
|
name = "the queue's policy grants pki/issue/swarm-nats and nothing else";
|
|
ok =
|
|
let
|
|
paths = map builtins.head (
|
|
builtins.filter builtins.isList (builtins.split "path \"([^\"]*)\"" policyScript)
|
|
);
|
|
in
|
|
paths == [ "pki/issue/swarm-nats" ]
|
|
&& lib.hasInfix ''capabilities = ["update"]'' policyScript
|
|
&& lib.hasInfix "allowed_common_names=swarm-nats" policyScript
|
|
&& lib.hasInfix "token_policies=swarm-nats" policyScript;
|
|
}
|
|
{
|
|
# Mint to consume: the leaf the unit writes is the one the server reads,
|
|
# and the login leaf glue-bao-tls signs is the one the unit presents.
|
|
name = "the server serves the leaf the host unit issues, and the unit logs in as swarm-nats";
|
|
ok =
|
|
natsTls.cert_file == "/var/lib/swarm-nats-tls/cert.pem"
|
|
&& natsTls.key_file == "/run/credentials/nats.service/tls-key"
|
|
&& lib.elem "tls-key:/var/lib/swarm-nats-tls/key.pem" natsContainer.systemd.services.nats.serviceConfig.LoadCredential
|
|
&& allLocal.containers.swarm-nats.bindMounts ? "/var/lib/swarm-nats-tls"
|
|
&& lib.hasInfix "d=/var/lib/swarm-nats-tls" leafUnit.script
|
|
&& lib.hasInfix "pki/issue/swarm-nats" leafUnit.script
|
|
&& leafUnit.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/nats.pem"
|
|
&& lib.hasInfix "[ -s /var/lib/swarm-bao-pki/nats.pem ]" allLocal.systemd.services.swarm-bao-pki.script
|
|
&& lib.hasInfix "swarm-nats \"\" clientAuth" allLocal.systemd.services.swarm-bao-pki.script;
|
|
}
|
|
{
|
|
name = "the server requires TLS: no allow_non_tls";
|
|
ok = !(natsContainer.services.nats.settings ? allow_non_tls);
|
|
}
|
|
{
|
|
name = "4222 is open on wg-hive when this host is on the mesh, and never host-wide";
|
|
ok =
|
|
lib.elem 4222 allLocalMesh.networking.firewall.interfaces.wg-hive.allowedTCPPorts
|
|
&& !(lib.elem 4222 allLocalMesh.networking.firewall.allowedTCPPorts)
|
|
&& lib.elem 4222 (bridgePorts allLocalMesh);
|
|
}
|
|
{
|
|
name = "4222 is not opened on wg-hive when this host is not on the mesh";
|
|
ok =
|
|
!(lib.elem 4222
|
|
(allLocal.networking.firewall.interfaces.wg-hive or { allowedTCPPorts = [ ]; }).allowedTCPPorts
|
|
)
|
|
&& !(lib.elem 4222 allLocal.networking.firewall.allowedTCPPorts);
|
|
}
|
|
{
|
|
# Ordering, never a requirement: the policy unit skips once the bootstrap
|
|
# token is gone, and a skipped unit counts as done.
|
|
name = "the leaf unit is ordered after its policy unit, with no requires";
|
|
ok =
|
|
let
|
|
p = "swarm-bao-nats-tls-policy.service";
|
|
in
|
|
lib.elem p leafUnit.after && lib.elem p leafUnit.wants && !(lib.elem p (leafUnit.requires or [ ]));
|
|
}
|
|
{
|
|
name = "a queue host whose store is elsewhere orders its leaf unit after no policy unit";
|
|
ok =
|
|
let
|
|
u = queueNoStore.systemd.services.swarm-bao-nats-tls;
|
|
in
|
|
!(lib.elem "swarm-bao-nats-tls-policy.service" u.after)
|
|
&& !(lib.elem "swarm-bao-nats-tls-policy.service" u.wants);
|
|
}
|
|
{
|
|
# Every hive dials the queue by the same name, so a hive away from it
|
|
# needs no URL of its own. Control and property in one: the scan must
|
|
# reach the controller and the agents' address here, and every URL it
|
|
# finds, like the options it reads through, is the name.
|
|
name = "a hive that is not the queue's host dials tls://<the queue's name>:4222 with nothing set";
|
|
ok =
|
|
let
|
|
s = clientUrls remote;
|
|
d = remote.services.hyperhive.deploy;
|
|
in
|
|
!d.nats.enable
|
|
&& s ? "host/hive-c0re/HIVE_AGENT_NATS_URL"
|
|
&& s ? "host/swarm-controller/SWARM_CONTROLLER_NATS_URL"
|
|
&& lib.all (u: u == natsUrl) (lib.attrValues s)
|
|
&& d.hive-controller.statusPublish.natsUrl == natsUrl
|
|
&& d.hive-controller.queue.agentNatsUrl == natsUrl
|
|
&& remote.services.hyperhive.swarm.controller.queue.natsUrl == natsUrl;
|
|
}
|
|
{
|
|
# The URL is set and the secret is not, which is every such hive until
|
|
# an operator places one: publishing is off, not misconfigured.
|
|
name = "that hive evaluates without an assertion failure";
|
|
ok = failedAssertions remote == [ ];
|
|
}
|
|
{
|
|
# Off means off: hive-c0re is handed no status coordinates, rather
|
|
# than a secret path nothing fills.
|
|
name = "without its status secret, that hive's hive-c0re is given no status coordinates";
|
|
ok =
|
|
let
|
|
s = remote.systemd.services.hive-c0re;
|
|
in
|
|
!(s.environment ? HIVE_C0RE_NATS_URL)
|
|
&& !(s.environment ? HIVE_C0RE_OIDC_CLIENT_SECRET_FILE)
|
|
&& !(lib.any (lib.hasPrefix "swarm-status-client.secret:") (
|
|
lib.toList (s.serviceConfig.LoadCredential or [ ])
|
|
));
|
|
}
|
|
{
|
|
# The secret alone turns publishing on, at the default URL.
|
|
name = "given its status secret, that hive publishes to the queue's name";
|
|
ok =
|
|
let
|
|
s = remotePublishing.systemd.services.hive-c0re;
|
|
in
|
|
failedAssertions remotePublishing == [ ]
|
|
&& s.environment.HIVE_C0RE_NATS_URL == natsUrl
|
|
&& s.environment.HIVE_C0RE_OIDC_CLIENT_SECRET_FILE == "%d/swarm-status-client.secret"
|
|
&& lib.elem "swarm-status-client.secret:/var/lib/secrets/hive-h1.secret" (
|
|
lib.toList s.serviceConfig.LoadCredential
|
|
);
|
|
}
|
|
{
|
|
# What the assertion was written for is still refused: a secret with
|
|
# nowhere to present it.
|
|
name = "a status secret without a queue URL is refused at eval";
|
|
ok = refusedStatusSecret remoteSecretNoUrl && !(refusedStatusSecret remote);
|
|
}
|
|
];
|
|
in
|
|
runGroup "nats-tls" cases
|