swarm-nats: give the queue a name, a bao-issued leaf, and require TLS
The queue listened in plaintext on 4222, reached by bridge IP or loopback,
and nothing in-tree opened it to another hive. It now has a name, serves a
certificate for that name alone, and refuses clients that do not speak TLS.
- `swarm.nats.domain`, default `nats.<swarm.domain>`, a sibling name like
`swarm.bao.domain`. The queue host answers it via `gateway.localNames`;
every other hive resolves it through the operator's DNS, as for bao.
- `pki/roles/swarm-nats` allows that one name (bare domain, no subdomains,
IPs or localhost, server flag). A `swarm-nats` cert-auth role and policy
may only `update` `pki/issue/swarm-nats`, written by
`swarm-bao-nats-tls-policy`. The login leaf is minted by glue-bao-tls and
paired by glue-nats-bao-identity. `deploy.bao.natsCommonName` is reserved
as a hive name.
- `swarm-bao-nats-tls` issues the leaf into a directory bound read-only into
the container, restarts nats when it rotates, and re-runs daily.
It joins glue-bao-readers-policy-order, so it is ordered after its policy
unit (`after` and `wants`, never `requires`) where the store is on the
same host. The policy unit joins the store's journald list.
- nats gets `tls {}`, with the key via `LoadCredential`, and no
`allow_non_tls`. `validateConfig` is now off in every mode, because the
build-time check loads a leaf that only exists at runtime.
- 4222 is also open on `wg-hive` when the host is on the mesh, never
host-wide.
- `statusPublish.natsUrl`, `queue.agentNatsUrl`, the controller's URL under
`singleHostSwarm`, and the auth responder all dial
`tls://<swarm.nats.domain>:<port>`. swarm-queue-client hands its CA file
to the NATS connection too, so hive-c0re and the controller trust the
leaf's root.
- docs/swarm/README.md: the queue URL and the one DNS record a multi-host
swarm needs.
module-eval-nats-tls pins the role, the policy, the served leaf, the
firewall, the ordering, and a scan of every `*_NATS_URL` and the
responder's URL across the host and its containers.
Closes #4626
This commit is contained in:
atlas
mara
parent
244db367c4
commit
1d261b3fed
17 changed files with 748 additions and 108 deletions
|
|
@ -123,19 +123,17 @@ let
|
|||
== toString allLocal.services.hyperhive.deploy.hive-controller.queue.agentCredentialDir;
|
||||
}
|
||||
{
|
||||
# The one address in this file that must NOT be loopback. Both spellings
|
||||
# sit in the same unit's environment and are correct for their own
|
||||
# reader: hive-c0re shares the host netns, an agent container does not,
|
||||
# so a copy-paste between them reaches the agent itself and the symptom
|
||||
# is a connect that hangs.
|
||||
name = "the agents' queue address is the bridge, not the loopback one the hive itself uses";
|
||||
# Inside an agent container loopback is the agent itself, so the agents'
|
||||
# address must never be one. By name it is the same string the hive
|
||||
# dials, which ./nats-tls.nix pins for every client.
|
||||
name = "the agents' queue address is the queue's name, never loopback";
|
||||
ok =
|
||||
let
|
||||
e = allLocal.systemd.services.hive-c0re.environment;
|
||||
in
|
||||
e.HIVE_AGENT_NATS_URL == "nats://${allLocal.services.hyperhive.network.bridgeIp}:4222"
|
||||
e.HIVE_AGENT_NATS_URL == "tls://nats.t.local:4222"
|
||||
&& !(lib.hasInfix "127.0.0.1" e.HIVE_AGENT_NATS_URL)
|
||||
&& e.HIVE_AGENT_NATS_URL != e.HIVE_C0RE_NATS_URL;
|
||||
&& e.HIVE_AGENT_NATS_URL == e.HIVE_C0RE_NATS_URL;
|
||||
}
|
||||
{
|
||||
# The agents mint against the swarm's IdP, the same endpoint the hive's
|
||||
|
|
|
|||
217
nix/module-eval/nats-tls.nix
Normal file
217
nix/module-eval/nats-tls.nix
Normal file
|
|
@ -0,0 +1,217 @@
|
|||
# `checks.module-eval-nats-tls` — see ./lib.nix for the shared rationale (why
|
||||
# this suite exists, naming convention, "evaluates not executes").
|
||||
#
|
||||
# The queue's name, its bao-issued leaf, and the clients that dial it.
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
self,
|
||||
nixosSystem,
|
||||
}:
|
||||
let
|
||||
inherit
|
||||
(import ./lib.nix {
|
||||
inherit
|
||||
pkgs
|
||||
lib
|
||||
self
|
||||
nixosSystem
|
||||
;
|
||||
})
|
||||
hive
|
||||
runGroup
|
||||
bridgePorts
|
||||
;
|
||||
|
||||
natsName = "nats.t.local";
|
||||
natsUrl = "tls://${natsName}:4222";
|
||||
|
||||
# Every service on one host, with a bootstrap token so the store's granting
|
||||
# units render. The queue, the store and every in-tree client of the queue
|
||||
# are all here, so the scan below reads each of them.
|
||||
allLocal = hive {
|
||||
deploy.singleHostSwarm = true;
|
||||
deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token";
|
||||
};
|
||||
|
||||
# The same host on the mesh.
|
||||
allLocalMesh = hive {
|
||||
deploy.singleHostSwarm = true;
|
||||
deploy.wireguard.enable = true;
|
||||
deploy.wireguard.address = "10.100.0.1/24";
|
||||
};
|
||||
|
||||
# The queue on a host whose store is elsewhere: no local policy unit.
|
||||
queueNoStore = hive {
|
||||
deploy.nats.enable = true;
|
||||
deploy.nats.autoGenerateCallout = true;
|
||||
};
|
||||
|
||||
policyScript = allLocal.systemd.services.swarm-bao-nats-tls-policy.script;
|
||||
leafUnit = allLocal.systemd.services.swarm-bao-nats-tls;
|
||||
natsContainer = allLocal.containers.swarm-nats.config;
|
||||
natsTls = natsContainer.services.nats.settings.tls;
|
||||
|
||||
# Every `(nats|tls)://…` in a string.
|
||||
urlsIn =
|
||||
s: map builtins.head (builtins.filter builtins.isList (builtins.split "((nats|tls)://[^ '\"]+)" s));
|
||||
|
||||
# Every in-tree queue client, found rather than listed. The Rust client reads
|
||||
# its address from `<PREFIX>_NATS_URL` (`swarm_queue_client::QueueConfig::
|
||||
# from_env`), so any unit on the host or in a container that is handed one
|
||||
# carries a variable of that shape. The responder takes a flag instead.
|
||||
# Keyed by where each came from, so the control below can name them.
|
||||
clientUrls =
|
||||
machine:
|
||||
let
|
||||
fromUnits =
|
||||
where: services:
|
||||
lib.concatLists (
|
||||
lib.mapAttrsToList (
|
||||
unit: s:
|
||||
lib.mapAttrsToList (var: v: {
|
||||
name = "${where}/${unit}/${var}";
|
||||
value = v;
|
||||
}) (lib.filterAttrs (var: v: lib.hasSuffix "_NATS_URL" var && v != null) (s.environment or { }))
|
||||
) services
|
||||
);
|
||||
containerUnits = lib.concatLists (
|
||||
lib.mapAttrsToList (c: cc: fromUnits c cc.config.systemd.services) machine.containers
|
||||
);
|
||||
responder =
|
||||
map
|
||||
(u: {
|
||||
name = "swarm-nats/swarm-nats-auth/--nats-url";
|
||||
value = u;
|
||||
})
|
||||
(
|
||||
urlsIn machine.containers.swarm-nats.config.systemd.services.swarm-nats-auth.serviceConfig.ExecStart
|
||||
);
|
||||
in
|
||||
lib.listToAttrs (fromUnits "host" machine.systemd.services ++ containerUnits ++ responder);
|
||||
|
||||
scanned = clientUrls allLocal;
|
||||
|
||||
cases = [
|
||||
{
|
||||
# Control first: a scan that found nothing would pass the next case
|
||||
# vacuously. These are the four clients in the tree today.
|
||||
name = "the client scan finds hive-c0re, the agents, the controller and the responder";
|
||||
ok = lib.all (k: scanned ? ${k}) [
|
||||
"host/hive-c0re/HIVE_C0RE_NATS_URL"
|
||||
"host/hive-c0re/HIVE_AGENT_NATS_URL"
|
||||
"host/swarm-controller/SWARM_CONTROLLER_NATS_URL"
|
||||
"swarm-nats/swarm-nats-auth/--nats-url"
|
||||
];
|
||||
}
|
||||
{
|
||||
# The server requires TLS and its leaf carries the name alone, so a
|
||||
# `nats://` URL or an address is a client that cannot connect. Every one
|
||||
# found, not the four above: a client added later is held to it too.
|
||||
name = "every in-tree queue client dials tls://<the queue's name>:4222";
|
||||
ok = lib.all (u: u == natsUrl) (lib.attrValues scanned);
|
||||
}
|
||||
{
|
||||
# The option defaults the scan reads through, so a client that stops
|
||||
# reading them does not also escape the property above.
|
||||
name = "the queue URL options default to the name on the queue's host";
|
||||
ok =
|
||||
let
|
||||
d = allLocal.services.hyperhive.deploy;
|
||||
in
|
||||
d.hive-controller.statusPublish.natsUrl == natsUrl
|
||||
&& d.hive-controller.queue.agentNatsUrl == natsUrl
|
||||
&& allLocal.services.hyperhive.swarm.controller.queue.natsUrl == natsUrl;
|
||||
}
|
||||
{
|
||||
name = "the queue's name is served by this host's resolver, at the bridge address";
|
||||
ok =
|
||||
lib.elem natsName allLocal.services.hyperhive.gateway.localNames
|
||||
&& lib.elem "/${natsName}/${allLocal.services.hyperhive.network.bridgeIp}" allLocal.services.dnsmasq.settings.address;
|
||||
}
|
||||
{
|
||||
name = "the queue's pki role issues for its name alone";
|
||||
ok = lib.all (arg: lib.hasInfix arg policyScript) [
|
||||
"roles/swarm-nats \\"
|
||||
"allowed_domains=${lib.escapeShellArg natsName} \\"
|
||||
"allow_bare_domains=true \\"
|
||||
"allow_subdomains=false \\"
|
||||
"allow_glob_domains=false \\"
|
||||
"allow_localhost=false \\"
|
||||
"allow_any_name=false \\"
|
||||
"allow_ip_sans=false \\"
|
||||
"server_flag=true \\"
|
||||
"client_flag=false \\"
|
||||
];
|
||||
}
|
||||
{
|
||||
# Every `path` the policy names, not a search for the one expected: a
|
||||
# second grant added later fails here.
|
||||
name = "the queue's policy grants pki/issue/swarm-nats and nothing else";
|
||||
ok =
|
||||
let
|
||||
paths = map builtins.head (
|
||||
builtins.filter builtins.isList (builtins.split "path \"([^\"]*)\"" policyScript)
|
||||
);
|
||||
in
|
||||
paths == [ "pki/issue/swarm-nats" ]
|
||||
&& lib.hasInfix ''capabilities = ["update"]'' policyScript
|
||||
&& lib.hasInfix "allowed_common_names=swarm-nats" policyScript
|
||||
&& lib.hasInfix "token_policies=swarm-nats" policyScript;
|
||||
}
|
||||
{
|
||||
# Mint to consume: the leaf the unit writes is the one the server reads,
|
||||
# and the login leaf glue-bao-tls signs is the one the unit presents.
|
||||
name = "the server serves the leaf the host unit issues, and the unit logs in as swarm-nats";
|
||||
ok =
|
||||
natsTls.cert_file == "/var/lib/swarm-nats-tls/cert.pem"
|
||||
&& natsTls.key_file == "/run/credentials/nats.service/tls-key"
|
||||
&& lib.elem "tls-key:/var/lib/swarm-nats-tls/key.pem" natsContainer.systemd.services.nats.serviceConfig.LoadCredential
|
||||
&& allLocal.containers.swarm-nats.bindMounts ? "/var/lib/swarm-nats-tls"
|
||||
&& lib.hasInfix "d=/var/lib/swarm-nats-tls" leafUnit.script
|
||||
&& lib.hasInfix "pki/issue/swarm-nats" leafUnit.script
|
||||
&& leafUnit.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/nats.pem"
|
||||
&& lib.hasInfix "[ -s /var/lib/swarm-bao-pki/nats.pem ]" allLocal.systemd.services.swarm-bao-pki.script
|
||||
&& lib.hasInfix "swarm-nats \"\" clientAuth" allLocal.systemd.services.swarm-bao-pki.script;
|
||||
}
|
||||
{
|
||||
name = "the server requires TLS: no allow_non_tls";
|
||||
ok = !(natsContainer.services.nats.settings ? allow_non_tls);
|
||||
}
|
||||
{
|
||||
name = "4222 is open on wg-hive when this host is on the mesh, and never host-wide";
|
||||
ok =
|
||||
lib.elem 4222 allLocalMesh.networking.firewall.interfaces.wg-hive.allowedTCPPorts
|
||||
&& !(lib.elem 4222 allLocalMesh.networking.firewall.allowedTCPPorts)
|
||||
&& lib.elem 4222 (bridgePorts allLocalMesh);
|
||||
}
|
||||
{
|
||||
name = "4222 is not opened on wg-hive when this host is not on the mesh";
|
||||
ok =
|
||||
!(lib.elem 4222
|
||||
(allLocal.networking.firewall.interfaces.wg-hive or { allowedTCPPorts = [ ]; }).allowedTCPPorts
|
||||
)
|
||||
&& !(lib.elem 4222 allLocal.networking.firewall.allowedTCPPorts);
|
||||
}
|
||||
{
|
||||
# Ordering, never a requirement: the policy unit skips once the bootstrap
|
||||
# token is gone, and a skipped unit counts as done.
|
||||
name = "the leaf unit is ordered after its policy unit, with no requires";
|
||||
ok =
|
||||
let
|
||||
p = "swarm-bao-nats-tls-policy.service";
|
||||
in
|
||||
lib.elem p leafUnit.after && lib.elem p leafUnit.wants && !(lib.elem p (leafUnit.requires or [ ]));
|
||||
}
|
||||
{
|
||||
name = "a queue host whose store is elsewhere orders its leaf unit after no policy unit";
|
||||
ok =
|
||||
let
|
||||
u = queueNoStore.systemd.services.swarm-bao-nats-tls;
|
||||
in
|
||||
!(lib.elem "swarm-bao-nats-tls-policy.service" u.after)
|
||||
&& !(lib.elem "swarm-bao-nats-tls-policy.service" u.wants);
|
||||
}
|
||||
];
|
||||
in
|
||||
runGroup "nats-tls" cases
|
||||
Loading…
Reference in a new issue