# `checks.module-eval-nats-tls` — see ./lib.nix for the shared rationale (why # this suite exists, naming convention, "evaluates not executes"). # # The queue's name, its bao-issued leaf, and the clients that dial it. { pkgs, lib, self, nixosSystem, }: let inherit (import ./lib.nix { inherit pkgs lib self nixosSystem ; }) hive runGroup bridgePorts ; natsName = "nats.t.local"; natsUrl = "tls://${natsName}:4222"; # Every service on one host. The queue, the store and every in-tree client of the queue # are all here, so the scan below reads each of them. allLocal = hive { deploy.singleHostSwarm = true; }; # The same host on the mesh. allLocalMesh = hive { deploy.singleHostSwarm = true; deploy.wireguard.enable = true; deploy.wireguard.address = "10.100.0.1/24"; }; # The queue on a host whose store is elsewhere: no local policy unit. queueNoStore = hive { deploy.nats.enable = true; deploy.nats.autoGenerateCallout = true; }; # A hive that is not the queue's host, with nothing about the queue's # address set by hand: what every hive but one in a multi-host swarm looks # like. The controller is on too, since it may run away from the queue. # # The two secrets are the ones a hive away from authelia already has to be # handed, and neither is an address; without them this hive would fail # assertions that have nothing to do with the queue. remoteSecrets = { deploy.forgejo.sso.clientSecretFile = "/var/lib/forgejo-oidc/by-hand.secret"; deploy.swarm-controller.queue.clientSecretFile = "/var/lib/secrets/swarm-controller.secret"; }; remote = hive (lib.recursiveUpdate remoteSecrets { deploy.swarm-controller.enable = true; }); # The same hive given its status secret by hand, which is what turns # publishing on away from the IdP's host. remotePublishing = hive ( lib.recursiveUpdate remoteSecrets { deploy.hive-controller.statusPublish.clientSecretFile = "/var/lib/secrets/hive-h1.secret"; } ); # A real half-config: the secret, with the URL it would be presented at # taken away. remoteSecretNoUrl = hive ( lib.recursiveUpdate remoteSecrets { deploy.hive-controller.statusPublish.clientSecretFile = "/var/lib/secrets/hive-h1.secret"; deploy.hive-controller.statusPublish.natsUrl = null; } ); failedAssertions = m: lib.filter (a: !a.assertion) m.assertions; refusedStatusSecret = m: lib.any (a: lib.hasInfix "status-publishing client secret" a.message) (failedAssertions m); policyScript = allLocal.systemd.services.swarm-bao-nats-tls-policy.script; leafUnit = allLocal.systemd.services.swarm-bao-nats-tls; natsContainer = allLocal.containers.swarm-nats.config; natsTls = natsContainer.services.nats.settings.tls; # Every `(nats|tls)://…` in a string. urlsIn = s: map builtins.head (builtins.filter builtins.isList (builtins.split "((nats|tls)://[^ '\"]+)" s)); # Every in-tree queue client, found rather than listed. The Rust client reads # its address from `_NATS_URL` (`swarm_queue_client::QueueConfig:: # from_env`), so any unit on the host or in a container that is handed one # carries a variable of that shape. The responder takes a flag instead. # Keyed by where each came from, so the control below can name them. clientUrls = machine: let fromUnits = where: services: lib.concatLists ( lib.mapAttrsToList ( unit: s: lib.mapAttrsToList (var: v: { name = "${where}/${unit}/${var}"; value = v; }) (lib.filterAttrs (var: v: lib.hasSuffix "_NATS_URL" var && v != null) (s.environment or { })) ) services ); containerUnits = lib.concatLists ( lib.mapAttrsToList (c: cc: fromUnits c cc.config.systemd.services) machine.containers ); # The responder runs beside the queue only, so a hive without one has # none to read. responder = map (u: { name = "swarm-nats/swarm-nats-auth/--nats-url"; value = u; }) ( lib.optionals (machine.containers ? swarm-nats) ( urlsIn machine.containers.swarm-nats.config.systemd.services.swarm-nats-auth.serviceConfig.ExecStart ) ); in lib.listToAttrs (fromUnits "host" machine.systemd.services ++ containerUnits ++ responder); scanned = clientUrls allLocal; cases = [ { # Control first: a scan that found nothing would pass the next case # vacuously. These are the four clients in the tree today. name = "the client scan finds hive-c0re, the agents, the controller and the responder"; ok = lib.all (k: scanned ? ${k}) [ "host/hive-c0re/HIVE_C0RE_NATS_URL" "host/hive-c0re/HIVE_AGENT_NATS_URL" "host/swarm-controller/SWARM_CONTROLLER_NATS_URL" "swarm-nats/swarm-nats-auth/--nats-url" ]; } { # The server requires TLS and its leaf carries the name alone, so a # `nats://` URL or an address is a client that cannot connect. Every one # found, not the four above: a client added later is held to it too. name = "every in-tree queue client dials tls://:4222"; ok = lib.all (u: u == natsUrl) (lib.attrValues scanned); } { # The option defaults the scan reads through, so a client that stops # reading them does not also escape the property above. name = "the queue URL options default to the name on the queue's host"; ok = let d = allLocal.services.hyperhive.deploy; in d.hive-controller.statusPublish.natsUrl == natsUrl && d.hive-controller.queue.agentNatsUrl == natsUrl && allLocal.services.hyperhive.swarm.controller.queue.natsUrl == natsUrl; } { name = "the queue's name is served by this host's resolver, at the bridge address"; ok = lib.elem natsName allLocal.services.hyperhive.gateway.localNames && lib.elem "/${natsName}/${allLocal.services.hyperhive.network.bridgeIp}" allLocal.services.dnsmasq.settings.address; } { name = "the queue's pki role issues for its name alone"; ok = lib.all (arg: lib.hasInfix arg policyScript) [ "roles/swarm-nats \\" "allowed_domains=${lib.escapeShellArg natsName} \\" "allow_bare_domains=true \\" "allow_subdomains=false \\" "allow_glob_domains=false \\" "allow_localhost=false \\" "allow_any_name=false \\" "allow_ip_sans=false \\" "server_flag=true \\" "client_flag=false \\" ]; } { # Every `path` the policy names, not a search for the one expected: a # second grant added later fails here. name = "the queue's policy grants pki/issue/swarm-nats and nothing else"; ok = let paths = map builtins.head ( builtins.filter builtins.isList (builtins.split "path \"([^\"]*)\"" policyScript) ); in paths == [ "pki/issue/swarm-nats" ] && lib.hasInfix ''capabilities = ["update"]'' policyScript && lib.hasInfix "allowed_common_names=swarm-nats" policyScript && lib.hasInfix "token_policies=swarm-nats" policyScript; } { # Mint to consume: the leaf the unit writes is the one the server reads, # and the login leaf glue-bao-tls signs is the one the unit presents. name = "the server serves the leaf the host unit issues, and the unit logs in as swarm-nats"; ok = natsTls.cert_file == "/var/lib/swarm-nats-tls/cert.pem" && natsTls.key_file == "/run/credentials/nats.service/tls-key" && lib.elem "tls-key:/var/lib/swarm-nats-tls/key.pem" natsContainer.systemd.services.nats.serviceConfig.LoadCredential && allLocal.containers.swarm-nats.bindMounts ? "/var/lib/swarm-nats-tls" && lib.hasInfix "d=/var/lib/swarm-nats-tls" leafUnit.script && lib.hasInfix "pki/issue/swarm-nats" leafUnit.script && leafUnit.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/nats.pem" && lib.hasInfix "[ -s /var/lib/swarm-bao-pki/nats.pem ]" allLocal.systemd.services.swarm-bao-pki.script && lib.hasInfix "swarm-nats \"\" clientAuth" allLocal.systemd.services.swarm-bao-pki.script; } { name = "the server requires TLS: no allow_non_tls"; ok = !(natsContainer.services.nats.settings ? allow_non_tls); } { name = "4222 is open on wg-hive when this host is on the mesh, and never host-wide"; ok = lib.elem 4222 allLocalMesh.networking.firewall.interfaces.wg-hive.allowedTCPPorts && !(lib.elem 4222 allLocalMesh.networking.firewall.allowedTCPPorts) && lib.elem 4222 (bridgePorts allLocalMesh); } { name = "4222 is not opened on wg-hive when this host is not on the mesh"; ok = !(lib.elem 4222 (allLocal.networking.firewall.interfaces.wg-hive or { allowedTCPPorts = [ ]; }).allowedTCPPorts ) && !(lib.elem 4222 allLocal.networking.firewall.allowedTCPPorts); } { # Ordering, never a requirement: a policy unit that failed still counts # as done, and the leaf unit's own retries carry it past that. name = "the leaf unit is ordered after its policy unit, with no requires"; ok = let p = "swarm-bao-nats-tls-policy.service"; in lib.elem p leafUnit.after && lib.elem p leafUnit.wants && !(lib.elem p (leafUnit.requires or [ ])); } { name = "a queue host whose store is elsewhere orders its leaf unit after no policy unit"; ok = let u = queueNoStore.systemd.services.swarm-bao-nats-tls; in !(lib.elem "swarm-bao-nats-tls-policy.service" u.after) && !(lib.elem "swarm-bao-nats-tls-policy.service" u.wants); } { # Every hive dials the queue by the same name, so a hive away from it # needs no URL of its own. Control and property in one: the scan must # reach the controller and the agents' address here, and every URL it # finds, like the options it reads through, is the name. name = "a hive that is not the queue's host dials tls://:4222 with nothing set"; ok = let s = clientUrls remote; d = remote.services.hyperhive.deploy; in !d.nats.enable && s ? "host/hive-c0re/HIVE_AGENT_NATS_URL" && s ? "host/swarm-controller/SWARM_CONTROLLER_NATS_URL" && lib.all (u: u == natsUrl) (lib.attrValues s) && d.hive-controller.statusPublish.natsUrl == natsUrl && d.hive-controller.queue.agentNatsUrl == natsUrl && remote.services.hyperhive.swarm.controller.queue.natsUrl == natsUrl; } { # The URL is set and the secret is not, which is every such hive until # an operator places one: publishing is off, not misconfigured. name = "that hive evaluates without an assertion failure"; ok = failedAssertions remote == [ ]; } { # Off means off: hive-c0re is handed no status coordinates, rather # than a secret path nothing fills. name = "without its status secret, that hive's hive-c0re is given no status coordinates"; ok = let s = remote.systemd.services.hive-c0re; in !(s.environment ? HIVE_C0RE_NATS_URL) && !(s.environment ? HIVE_C0RE_OIDC_CLIENT_SECRET_FILE) && !(lib.any (lib.hasPrefix "swarm-status-client.secret:") ( lib.toList (s.serviceConfig.LoadCredential or [ ]) )); } { # The secret alone turns publishing on, at the default URL. name = "given its status secret, that hive publishes to the queue's name"; ok = let s = remotePublishing.systemd.services.hive-c0re; in failedAssertions remotePublishing == [ ] && s.environment.HIVE_C0RE_NATS_URL == natsUrl && s.environment.HIVE_C0RE_OIDC_CLIENT_SECRET_FILE == "%d/swarm-status-client.secret" && lib.elem "swarm-status-client.secret:/var/lib/secrets/hive-h1.secret" ( lib.toList s.serviceConfig.LoadCredential ); } { # What the assertion was written for is still refused: a secret with # nowhere to present it. name = "a status secret without a queue URL is refused at eval"; ok = refusedStatusSecret remoteSecretNoUrl && !(refusedStatusSecret remote); } ]; in runGroup "nats-tls" cases