5761398d60
swarm-services-cert issues the swarm-services leaf a self-signed gateway
serves, and on a host fronting no swarm name fetches the store's pki root.
It is gated on the gateway, not on a hive, yet its leaf pair was named
under deploy.hive-controller.tls and defaulted from the hive reader's cert
directory, and the root fetch presented the hive's own identity. A gateway
host with no hive had no such identity, so that branch was red by
construction.
- The pair moves to deploy.bao.servicesIssuerClient{Cert,Key}File, beside
the store's other host-unit readers, and glue-bao-tls.nix defaults it on
the store's host from the leaf it already mints. The old
deploy.hive-controller.tls.baoClient{Cert,Key}File paths are
mkRenamedOptionModule aliases. glue-services-issuer-bao-identity.nix,
which derived the default from deploy.bao.clientCertFile, is gone.
- The no-swarm-name root fetch presents the services-issuer pair. The hive
identity is now only hive-tls-resign's.
- A self-signed gateway with no pair fails evaluation, naming both options.
A pair named but unreadable still fails the unit, naming the paths.
Refs #4921
364 lines
15 KiB
Nix
364 lines
15 KiB
Nix
# `checks.module-eval-nats-tls` — see ./lib.nix for the shared rationale (why
|
|
# this suite exists, naming convention, "evaluates not executes").
|
|
#
|
|
# The queue's name, its bao-issued leaf, and the clients that dial it.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
hiveWithController
|
|
runGroup
|
|
bridgePorts
|
|
;
|
|
|
|
natsName = "nats.t.local";
|
|
natsUrl = "tls://${natsName}:4222";
|
|
|
|
# Every service on one host. The queue, the store and every in-tree client of the queue
|
|
# are all here, so the scan below reads each of them.
|
|
allLocal = hiveWithController {
|
|
deploy.singleHostSwarm = true;
|
|
};
|
|
|
|
# The same host on the mesh.
|
|
allLocalMesh = hive {
|
|
deploy.singleHostSwarm = true;
|
|
deploy.wireguard.enable = true;
|
|
deploy.wireguard.address = "10.100.0.1/24";
|
|
};
|
|
|
|
# The queue on a host whose store is elsewhere: no local policy unit.
|
|
queueNoStore = hive {
|
|
deploy.nats.enable = true;
|
|
};
|
|
|
|
# The store's host with the hive's own reader pointed somewhere else.
|
|
natsOnStoreHostReaderMoved = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.nats.enable = true;
|
|
deploy.nats.autoGenerateCallout = true;
|
|
deploy.bao.clientCertFile = "/etc/pki/h1.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem";
|
|
};
|
|
|
|
# A queue host holding only the hive's own store reader, placed by hand: no
|
|
# store here, and no queue leaf named.
|
|
queueWithHiveReaderOnly = hive {
|
|
deploy.nats.enable = true;
|
|
deploy.nats.autoGenerateCallout = true;
|
|
deploy.bao.clientCertFile = "/etc/pki/h1.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem";
|
|
};
|
|
|
|
# A hive that is not the queue's host, with nothing about the queue's
|
|
# address set by hand: what every hive but one in a multi-host swarm looks
|
|
# like. The controller is on too, since it may run away from the queue.
|
|
#
|
|
# The forge's secret, the controller's store identity and the gateway's
|
|
# services-issuer pair are what a hive away from authelia and the store
|
|
# already has to be handed, and none is an address; without them this hive
|
|
# would fail assertions that have nothing to do with the queue.
|
|
remoteSecrets = {
|
|
deploy.forgejo.sso.clientSecretFile = "/var/lib/forgejo-oidc/by-hand.secret";
|
|
deploy.swarm-controller.baoClientCertFile = "/var/lib/swarm-controller/bao-client.pem";
|
|
deploy.swarm-controller.baoClientKeyFile = "/var/lib/swarm-controller/bao-client-key.pem";
|
|
deploy.bao.servicesIssuerClientCertFile = "/etc/pki/bao-services-issuer.pem";
|
|
deploy.bao.servicesIssuerClientKeyFile = "/etc/pki/bao-services-issuer-key.pem";
|
|
};
|
|
remote = hiveWithController (
|
|
lib.recursiveUpdate remoteSecrets { deploy.swarm-controller.enable = true; }
|
|
);
|
|
|
|
# The same hive given its status secret by hand, which is what turns
|
|
# publishing on away from the IdP's host.
|
|
remotePublishing = hiveWithController (
|
|
lib.recursiveUpdate remoteSecrets {
|
|
deploy.hive-controller.statusPublish.clientSecretFile = "/var/lib/secrets/hive-h1.secret";
|
|
}
|
|
);
|
|
|
|
# A real half-config: the secret, with the URL it would be presented at
|
|
# taken away.
|
|
remoteSecretNoUrl = hive (
|
|
lib.recursiveUpdate remoteSecrets {
|
|
deploy.hive-controller.statusPublish.clientSecretFile = "/var/lib/secrets/hive-h1.secret";
|
|
deploy.hive-controller.statusPublish.natsUrl = null;
|
|
}
|
|
);
|
|
|
|
failedAssertions = m: lib.filter (a: !a.assertion) m.assertions;
|
|
refusedStatusSecret =
|
|
m: lib.any (a: lib.hasInfix "status-publishing client secret" a.message) (failedAssertions m);
|
|
|
|
policyScript = allLocal.systemd.services.swarm-bao-nats-tls-policy.script;
|
|
leafUnit = allLocal.systemd.services.swarm-bao-nats-tls;
|
|
natsContainer = allLocal.containers.swarm-nats.config;
|
|
natsTls = natsContainer.services.nats.settings.tls;
|
|
|
|
# Every `(nats|tls)://…` in a string.
|
|
urlsIn =
|
|
s: map builtins.head (builtins.filter builtins.isList (builtins.split "((nats|tls)://[^ '\"]+)" s));
|
|
|
|
# Every in-tree queue client, found rather than listed. The Rust client reads
|
|
# its address from `<PREFIX>_NATS_URL` (`swarm_queue_client::QueueConfig::
|
|
# from_env`), so any unit on the host or in a container that is handed one
|
|
# carries a variable of that shape. The responder takes a flag instead.
|
|
# Keyed by where each came from, so the control below can name them.
|
|
clientUrls =
|
|
machine:
|
|
let
|
|
fromUnits =
|
|
where: services:
|
|
lib.concatLists (
|
|
lib.mapAttrsToList (
|
|
unit: s:
|
|
lib.mapAttrsToList (var: v: {
|
|
name = "${where}/${unit}/${var}";
|
|
value = v;
|
|
}) (lib.filterAttrs (var: v: lib.hasSuffix "_NATS_URL" var && v != null) (s.environment or { }))
|
|
) services
|
|
);
|
|
containerUnits = lib.concatLists (
|
|
lib.mapAttrsToList (c: cc: fromUnits c cc.config.systemd.services) machine.containers
|
|
);
|
|
# The responder runs beside the queue only, so a hive without one has
|
|
# none to read.
|
|
responder =
|
|
map
|
|
(u: {
|
|
name = "swarm-nats/swarm-nats-auth/--nats-url";
|
|
value = u;
|
|
})
|
|
(
|
|
lib.optionals (machine.containers ? swarm-nats) (
|
|
urlsIn machine.containers.swarm-nats.config.systemd.services.swarm-nats-auth.serviceConfig.ExecStart
|
|
)
|
|
);
|
|
in
|
|
lib.listToAttrs (fromUnits "host" machine.systemd.services ++ containerUnits ++ responder);
|
|
|
|
scanned = clientUrls allLocal;
|
|
|
|
cases = [
|
|
{
|
|
# Control first: a scan that found nothing would pass the next case
|
|
# vacuously. These are the four clients in the tree today.
|
|
name = "the client scan finds hive-c0re, the agents, the controller and the responder";
|
|
ok = lib.all (k: scanned ? ${k}) [
|
|
"host/hive-c0re/HIVE_C0RE_NATS_URL"
|
|
"host/hive-c0re/HIVE_AGENT_NATS_URL"
|
|
"host/swarm-controller/SWARM_CONTROLLER_NATS_URL"
|
|
"swarm-nats/swarm-nats-auth/--nats-url"
|
|
];
|
|
}
|
|
{
|
|
# The server requires TLS and its leaf carries the name alone, so a
|
|
# `nats://` URL or an address is a client that cannot connect. Every one
|
|
# found, not the four above: a client added later is held to it too.
|
|
name = "every in-tree queue client dials tls://<the queue's name>:4222";
|
|
ok = lib.all (u: u == natsUrl) (lib.attrValues scanned);
|
|
}
|
|
{
|
|
# The option defaults the scan reads through, so a client that stops
|
|
# reading them does not also escape the property above.
|
|
name = "the queue URL options default to the name on the queue's host";
|
|
ok =
|
|
let
|
|
d = allLocal.services.hyperhive.deploy;
|
|
in
|
|
d.hive-controller.statusPublish.natsUrl == natsUrl
|
|
&& d.hive-controller.queue.agentNatsUrl == natsUrl
|
|
&& allLocal.services.hyperhive.swarm.controller.queue.natsUrl == natsUrl;
|
|
}
|
|
{
|
|
name = "the queue's name is served by this host's resolver, at the bridge address";
|
|
ok =
|
|
lib.elem natsName allLocal.services.hyperhive.gateway.localNames
|
|
&& lib.elem "/${natsName}/${allLocal.services.hyperhive.network.bridgeIp}" allLocal.services.dnsmasq.settings.address;
|
|
}
|
|
{
|
|
name = "the queue's pki role issues for its name alone";
|
|
ok = lib.all (arg: lib.hasInfix arg policyScript) [
|
|
"roles/swarm-nats \\"
|
|
"allowed_domains=${lib.escapeShellArg natsName} \\"
|
|
"allow_bare_domains=true \\"
|
|
"allow_subdomains=false \\"
|
|
"allow_glob_domains=false \\"
|
|
"allow_localhost=false \\"
|
|
"allow_any_name=false \\"
|
|
"allow_ip_sans=false \\"
|
|
"server_flag=true \\"
|
|
"client_flag=false \\"
|
|
];
|
|
}
|
|
{
|
|
# Every `path` the policy names, not a search for the one expected: a
|
|
# second grant added later fails here.
|
|
name = "the queue's policy grants pki/issue/swarm-nats and nothing else";
|
|
ok =
|
|
let
|
|
paths = map builtins.head (
|
|
builtins.filter builtins.isList (builtins.split "path \"([^\"]*)\"" policyScript)
|
|
);
|
|
in
|
|
paths == [ "pki/issue/swarm-nats" ]
|
|
&& lib.hasInfix ''capabilities = ["update"]'' policyScript
|
|
&& lib.hasInfix "allowed_common_names=swarm-nats" policyScript
|
|
&& lib.hasInfix "token_policies=swarm-nats" policyScript;
|
|
}
|
|
{
|
|
# Mint to consume: the leaf the unit writes is the one the server reads,
|
|
# and the login leaf glue-bao-tls signs is the one the unit presents.
|
|
name = "the server serves the leaf the host unit issues, and the unit logs in as swarm-nats";
|
|
ok =
|
|
natsTls.cert_file == "/var/lib/swarm-nats-tls/cert.pem"
|
|
&& natsTls.key_file == "/run/credentials/nats.service/tls-key"
|
|
&& lib.elem "tls-key:/var/lib/swarm-nats-tls/key.pem" natsContainer.systemd.services.nats.serviceConfig.LoadCredential
|
|
&& allLocal.containers.swarm-nats.bindMounts ? "/var/lib/swarm-nats-tls"
|
|
&& lib.hasInfix "d=/var/lib/swarm-nats-tls" leafUnit.script
|
|
&& lib.hasInfix "pki/issue/swarm-nats" leafUnit.script
|
|
&& leafUnit.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/nats.pem"
|
|
&& lib.hasInfix "[ -s /var/lib/swarm-bao-pki/nats.pem ]" allLocal.systemd.services.swarm-bao-pki.script
|
|
&& lib.hasInfix "swarm-nats \"\" clientAuth" allLocal.systemd.services.swarm-bao-pki.script;
|
|
}
|
|
{
|
|
# The hive's own reader is not the queue's identity, and no path is
|
|
# derived from it: a queue host without the store that holds only the
|
|
# hive's reader gets the leaf unit that fails naming the options, like
|
|
# one holding nothing.
|
|
name = "a queue host without the store is refused when it holds only the hive's reader";
|
|
ok =
|
|
let
|
|
u = queueWithHiveReaderOnly.systemd.services.swarm-bao-nats-tls;
|
|
in
|
|
queueWithHiveReaderOnly.services.hyperhive.deploy.nats.baoClientCertFile == null
|
|
&& !(u.environment ? BAO_CLIENT_CERT)
|
|
&& lib.hasInfix "services.hyperhive.deploy.nats.baoClient{Cert,Key}File" u.script;
|
|
}
|
|
{
|
|
# The default is the file the minting unit writes, wherever the hive's
|
|
# own reader is pointed.
|
|
name = "on the store's host the queue's leaf is the one minted there, not one beside the hive's reader";
|
|
ok =
|
|
let
|
|
n = natsOnStoreHostReaderMoved.services.hyperhive.deploy.nats;
|
|
mint = natsOnStoreHostReaderMoved.systemd.services.swarm-bao-pki.script;
|
|
in
|
|
n.baoClientCertFile == "/var/lib/swarm-bao-pki/nats.pem"
|
|
&& n.baoClientKeyFile == "/var/lib/swarm-bao-pki/nats-key.pem"
|
|
&& lib.hasInfix "[ -s /var/lib/swarm-bao-pki/nats.pem ] ||" mint
|
|
&& lib.hasInfix "/var/lib/swarm-bao-pki nats " mint;
|
|
}
|
|
{
|
|
name = "the server requires TLS: no allow_non_tls";
|
|
ok = !(natsContainer.services.nats.settings ? allow_non_tls);
|
|
}
|
|
{
|
|
name = "4222 is open on wg-hive when this host is on the mesh, and never host-wide";
|
|
ok =
|
|
lib.elem 4222 allLocalMesh.networking.firewall.interfaces.wg-hive.allowedTCPPorts
|
|
&& !(lib.elem 4222 allLocalMesh.networking.firewall.allowedTCPPorts)
|
|
&& lib.elem 4222 (bridgePorts allLocalMesh);
|
|
}
|
|
{
|
|
name = "4222 is not opened on wg-hive when this host is not on the mesh";
|
|
ok =
|
|
!(lib.elem 4222
|
|
(allLocal.networking.firewall.interfaces.wg-hive or { allowedTCPPorts = [ ]; }).allowedTCPPorts
|
|
)
|
|
&& !(lib.elem 4222 allLocal.networking.firewall.allowedTCPPorts);
|
|
}
|
|
{
|
|
# Ordering, never a requirement: a policy unit that failed still counts
|
|
# as done, and the leaf unit's own retries carry it past that.
|
|
name = "the leaf unit is ordered after its policy unit, with no requires";
|
|
ok =
|
|
let
|
|
p = "swarm-bao-nats-tls-policy.service";
|
|
in
|
|
lib.elem p leafUnit.after && lib.elem p leafUnit.wants && !(lib.elem p (leafUnit.requires or [ ]));
|
|
}
|
|
{
|
|
name = "a queue host whose store is elsewhere orders its leaf unit after no policy unit";
|
|
ok =
|
|
let
|
|
u = queueNoStore.systemd.services.swarm-bao-nats-tls;
|
|
in
|
|
!(lib.elem "swarm-bao-nats-tls-policy.service" u.after)
|
|
&& !(lib.elem "swarm-bao-nats-tls-policy.service" u.wants);
|
|
}
|
|
{
|
|
# Every hive dials the queue by the same name, so a hive away from it
|
|
# needs no URL of its own. Control and property in one: the scan must
|
|
# reach the controller and the agents' address here, and every URL it
|
|
# finds, like the options it reads through, is the name.
|
|
name = "a hive that is not the queue's host dials tls://<the queue's name>:4222 with nothing set";
|
|
ok =
|
|
let
|
|
s = clientUrls remote;
|
|
d = remote.services.hyperhive.deploy;
|
|
in
|
|
!d.nats.enable
|
|
&& s ? "host/hive-c0re/HIVE_AGENT_NATS_URL"
|
|
&& s ? "host/swarm-controller/SWARM_CONTROLLER_NATS_URL"
|
|
&& lib.all (u: u == natsUrl) (lib.attrValues s)
|
|
&& d.hive-controller.statusPublish.natsUrl == natsUrl
|
|
&& d.hive-controller.queue.agentNatsUrl == natsUrl
|
|
&& remote.services.hyperhive.swarm.controller.queue.natsUrl == natsUrl;
|
|
}
|
|
{
|
|
# The URL is set and the secret is not, which is every such hive until
|
|
# an operator places one: publishing is off, not misconfigured.
|
|
name = "that hive evaluates without an assertion failure";
|
|
ok = failedAssertions remote == [ ];
|
|
}
|
|
{
|
|
# Off means off: hive-c0re is handed no status coordinates, rather
|
|
# than a secret path nothing fills.
|
|
name = "without its status secret, that hive's hive-c0re is given no status coordinates";
|
|
ok =
|
|
let
|
|
s = remote.systemd.services.hive-c0re;
|
|
in
|
|
!(s.environment ? HIVE_C0RE_NATS_URL)
|
|
&& !(s.environment ? HIVE_C0RE_OIDC_CLIENT_SECRET_FILE)
|
|
&& !(lib.any (lib.hasPrefix "swarm-status-client.secret:") (
|
|
lib.toList (s.serviceConfig.LoadCredential or [ ])
|
|
));
|
|
}
|
|
{
|
|
# The secret alone turns publishing on, at the default URL.
|
|
name = "given its status secret, that hive publishes to the queue's name";
|
|
ok =
|
|
let
|
|
s = remotePublishing.systemd.services.hive-c0re;
|
|
in
|
|
failedAssertions remotePublishing == [ ]
|
|
&& s.environment.HIVE_C0RE_NATS_URL == natsUrl
|
|
&& s.environment.HIVE_C0RE_OIDC_CLIENT_SECRET_FILE == "%d/swarm-status-client.secret"
|
|
&& lib.elem "swarm-status-client.secret:/var/lib/secrets/hive-h1.secret" (
|
|
lib.toList s.serviceConfig.LoadCredential
|
|
);
|
|
}
|
|
{
|
|
# What the assertion was written for is still refused: a secret with
|
|
# nowhere to present it.
|
|
name = "a status secret without a queue URL is refused at eval";
|
|
ok = refusedStatusSecret remoteSecretNoUrl && !(refusedStatusSecret remote);
|
|
}
|
|
];
|
|
in
|
|
runGroup "nats-tls" cases
|