hyperhive/nix/module-eval/nats-tls.nix

# `checks.module-eval-nats-tls` — see ./lib.nix for the shared rationale (why
# this suite exists, naming convention, "evaluates not executes").
#
# The queue's name, its bao-issued leaf, and the clients that dial it.
{
  pkgs,
  lib,
  self,
  nixosSystem,
}:
let
  inherit
    (import ./lib.nix {
      inherit
        pkgs
        lib
        self
        nixosSystem
        ;
    })
    hive
    runGroup
    bridgePorts
    ;

  natsName = "nats.t.local";
  natsUrl = "tls://${natsName}:4222";

  # Every service on one host. The queue, the store and every in-tree client of the queue
  # are all here, so the scan below reads each of them.
  allLocal = hive {
    deploy.singleHostSwarm = true;
  };

  # The same host on the mesh.
  allLocalMesh = hive {
    deploy.singleHostSwarm = true;
    deploy.wireguard.enable = true;
    deploy.wireguard.address = "10.100.0.1/24";
  };

  # The queue on a host whose store is elsewhere: no local policy unit.
  queueNoStore = hive {
    deploy.nats.enable = true;
    deploy.nats.autoGenerateCallout = true;
  };

  # A hive that is not the queue's host, with nothing about the queue's
  # address set by hand: what every hive but one in a multi-host swarm looks
  # like. The controller is on too, since it may run away from the queue.
  #
  # The two secrets are the ones a hive away from authelia already has to be
  # handed, and neither is an address; without them this hive would fail
  # assertions that have nothing to do with the queue.
  remoteSecrets = {
    deploy.forgejo.sso.clientSecretFile = "/var/lib/forgejo-oidc/by-hand.secret";
    deploy.swarm-controller.queue.clientSecretFile = "/var/lib/secrets/swarm-controller.secret";
  };
  remote = hive (lib.recursiveUpdate remoteSecrets { deploy.swarm-controller.enable = true; });

  # The same hive given its status secret by hand, which is what turns
  # publishing on away from the IdP's host.
  remotePublishing = hive (
    lib.recursiveUpdate remoteSecrets {
      deploy.hive-controller.statusPublish.clientSecretFile = "/var/lib/secrets/hive-h1.secret";
    }
  );

  # A real half-config: the secret, with the URL it would be presented at
  # taken away.
  remoteSecretNoUrl = hive (
    lib.recursiveUpdate remoteSecrets {
      deploy.hive-controller.statusPublish.clientSecretFile = "/var/lib/secrets/hive-h1.secret";
      deploy.hive-controller.statusPublish.natsUrl = null;
    }
  );

  failedAssertions = m: lib.filter (a: !a.assertion) m.assertions;
  refusedStatusSecret =
    m: lib.any (a: lib.hasInfix "status-publishing client secret" a.message) (failedAssertions m);

  policyScript = allLocal.systemd.services.swarm-bao-nats-tls-policy.script;
  leafUnit = allLocal.systemd.services.swarm-bao-nats-tls;
  natsContainer = allLocal.containers.swarm-nats.config;
  natsTls = natsContainer.services.nats.settings.tls;

  # Every `(nats|tls)://…` in a string.
  urlsIn =
    s: map builtins.head (builtins.filter builtins.isList (builtins.split "((nats|tls)://[^ '\"]+)" s));

  # Every in-tree queue client, found rather than listed. The Rust client reads
  # its address from `<PREFIX>_NATS_URL` (`swarm_queue_client::QueueConfig::
  # from_env`), so any unit on the host or in a container that is handed one
  # carries a variable of that shape. The responder takes a flag instead.
  # Keyed by where each came from, so the control below can name them.
  clientUrls =
    machine:
    let
      fromUnits =
        where: services:
        lib.concatLists (
          lib.mapAttrsToList (
            unit: s:
            lib.mapAttrsToList (var: v: {
              name = "${where}/${unit}/${var}";
              value = v;
            }) (lib.filterAttrs (var: v: lib.hasSuffix "_NATS_URL" var && v != null) (s.environment or { }))
          ) services
        );
      containerUnits = lib.concatLists (
        lib.mapAttrsToList (c: cc: fromUnits c cc.config.systemd.services) machine.containers
      );
      # The responder runs beside the queue only, so a hive without one has
      # none to read.
      responder =
        map
          (u: {
            name = "swarm-nats/swarm-nats-auth/--nats-url";
            value = u;
          })
          (
            lib.optionals (machine.containers ? swarm-nats) (
              urlsIn machine.containers.swarm-nats.config.systemd.services.swarm-nats-auth.serviceConfig.ExecStart
            )
          );
    in
    lib.listToAttrs (fromUnits "host" machine.systemd.services ++ containerUnits ++ responder);

  scanned = clientUrls allLocal;

  cases = [
    {
      # Control first: a scan that found nothing would pass the next case
      # vacuously. These are the four clients in the tree today.
      name = "the client scan finds hive-c0re, the agents, the controller and the responder";
      ok = lib.all (k: scanned ? ${k}) [
        "host/hive-c0re/HIVE_C0RE_NATS_URL"
        "host/hive-c0re/HIVE_AGENT_NATS_URL"
        "host/swarm-controller/SWARM_CONTROLLER_NATS_URL"
        "swarm-nats/swarm-nats-auth/--nats-url"
      ];
    }
    {
      # The server requires TLS and its leaf carries the name alone, so a
      # `nats://` URL or an address is a client that cannot connect. Every one
      # found, not the four above: a client added later is held to it too.
      name = "every in-tree queue client dials tls://<the queue's name>:4222";
      ok = lib.all (u: u == natsUrl) (lib.attrValues scanned);
    }
    {
      # The option defaults the scan reads through, so a client that stops
      # reading them does not also escape the property above.
      name = "the queue URL options default to the name on the queue's host";
      ok =
        let
          d = allLocal.services.hyperhive.deploy;
        in
        d.hive-controller.statusPublish.natsUrl == natsUrl
        && d.hive-controller.queue.agentNatsUrl == natsUrl
        && allLocal.services.hyperhive.swarm.controller.queue.natsUrl == natsUrl;
    }
    {
      name = "the queue's name is served by this host's resolver, at the bridge address";
      ok =
        lib.elem natsName allLocal.services.hyperhive.gateway.localNames
        && lib.elem "/${natsName}/${allLocal.services.hyperhive.network.bridgeIp}" allLocal.services.dnsmasq.settings.address;
    }
    {
      name = "the queue's pki role issues for its name alone";
      ok = lib.all (arg: lib.hasInfix arg policyScript) [
        "roles/swarm-nats \\"
        "allowed_domains=${lib.escapeShellArg natsName} \\"
        "allow_bare_domains=true \\"
        "allow_subdomains=false \\"
        "allow_glob_domains=false \\"
        "allow_localhost=false \\"
        "allow_any_name=false \\"
        "allow_ip_sans=false \\"
        "server_flag=true \\"
        "client_flag=false \\"
      ];
    }
    {
      # Every `path` the policy names, not a search for the one expected: a
      # second grant added later fails here.
      name = "the queue's policy grants pki/issue/swarm-nats and nothing else";
      ok =
        let
          paths = map builtins.head (
            builtins.filter builtins.isList (builtins.split "path \"([^\"]*)\"" policyScript)
          );
        in
        paths == [ "pki/issue/swarm-nats" ]
        && lib.hasInfix ''capabilities = ["update"]'' policyScript
        && lib.hasInfix "allowed_common_names=swarm-nats" policyScript
        && lib.hasInfix "token_policies=swarm-nats" policyScript;
    }
    {
      # Mint to consume: the leaf the unit writes is the one the server reads,
      # and the login leaf glue-bao-tls signs is the one the unit presents.
      name = "the server serves the leaf the host unit issues, and the unit logs in as swarm-nats";
      ok =
        natsTls.cert_file == "/var/lib/swarm-nats-tls/cert.pem"
        && natsTls.key_file == "/run/credentials/nats.service/tls-key"
        && lib.elem "tls-key:/var/lib/swarm-nats-tls/key.pem" natsContainer.systemd.services.nats.serviceConfig.LoadCredential
        && allLocal.containers.swarm-nats.bindMounts ? "/var/lib/swarm-nats-tls"
        && lib.hasInfix "d=/var/lib/swarm-nats-tls" leafUnit.script
        && lib.hasInfix "pki/issue/swarm-nats" leafUnit.script
        && leafUnit.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/nats.pem"
        && lib.hasInfix "[ -s /var/lib/swarm-bao-pki/nats.pem ]" allLocal.systemd.services.swarm-bao-pki.script
        && lib.hasInfix "swarm-nats \"\" clientAuth" allLocal.systemd.services.swarm-bao-pki.script;
    }
    {
      name = "the server requires TLS: no allow_non_tls";
      ok = !(natsContainer.services.nats.settings ? allow_non_tls);
    }
    {
      name = "4222 is open on wg-hive when this host is on the mesh, and never host-wide";
      ok =
        lib.elem 4222 allLocalMesh.networking.firewall.interfaces.wg-hive.allowedTCPPorts
        && !(lib.elem 4222 allLocalMesh.networking.firewall.allowedTCPPorts)
        && lib.elem 4222 (bridgePorts allLocalMesh);
    }
    {
      name = "4222 is not opened on wg-hive when this host is not on the mesh";
      ok =
        !(lib.elem 4222
          (allLocal.networking.firewall.interfaces.wg-hive or { allowedTCPPorts = [ ]; }).allowedTCPPorts
        )
        && !(lib.elem 4222 allLocal.networking.firewall.allowedTCPPorts);
    }
    {
      # Ordering, never a requirement: a policy unit that failed still counts
      # as done, and the leaf unit's own retries carry it past that.
      name = "the leaf unit is ordered after its policy unit, with no requires";
      ok =
        let
          p = "swarm-bao-nats-tls-policy.service";
        in
        lib.elem p leafUnit.after && lib.elem p leafUnit.wants && !(lib.elem p (leafUnit.requires or [ ]));
    }
    {
      name = "a queue host whose store is elsewhere orders its leaf unit after no policy unit";
      ok =
        let
          u = queueNoStore.systemd.services.swarm-bao-nats-tls;
        in
        !(lib.elem "swarm-bao-nats-tls-policy.service" u.after)
        && !(lib.elem "swarm-bao-nats-tls-policy.service" u.wants);
    }
    {
      # Every hive dials the queue by the same name, so a hive away from it
      # needs no URL of its own. Control and property in one: the scan must
      # reach the controller and the agents' address here, and every URL it
      # finds, like the options it reads through, is the name.
      name = "a hive that is not the queue's host dials tls://<the queue's name>:4222 with nothing set";
      ok =
        let
          s = clientUrls remote;
          d = remote.services.hyperhive.deploy;
        in
        !d.nats.enable
        && s ? "host/hive-c0re/HIVE_AGENT_NATS_URL"
        && s ? "host/swarm-controller/SWARM_CONTROLLER_NATS_URL"
        && lib.all (u: u == natsUrl) (lib.attrValues s)
        && d.hive-controller.statusPublish.natsUrl == natsUrl
        && d.hive-controller.queue.agentNatsUrl == natsUrl
        && remote.services.hyperhive.swarm.controller.queue.natsUrl == natsUrl;
    }
    {
      # The URL is set and the secret is not, which is every such hive until
      # an operator places one: publishing is off, not misconfigured.
      name = "that hive evaluates without an assertion failure";
      ok = failedAssertions remote == [ ];
    }
    {
      # Off means off: hive-c0re is handed no status coordinates, rather
      # than a secret path nothing fills.
      name = "without its status secret, that hive's hive-c0re is given no status coordinates";
      ok =
        let
          s = remote.systemd.services.hive-c0re;
        in
        !(s.environment ? HIVE_C0RE_NATS_URL)
        && !(s.environment ? HIVE_C0RE_OIDC_CLIENT_SECRET_FILE)
        && !(lib.any (lib.hasPrefix "swarm-status-client.secret:") (
          lib.toList (s.serviceConfig.LoadCredential or [ ])
        ));
    }
    {
      # The secret alone turns publishing on, at the default URL.
      name = "given its status secret, that hive publishes to the queue's name";
      ok =
        let
          s = remotePublishing.systemd.services.hive-c0re;
        in
        failedAssertions remotePublishing == [ ]
        && s.environment.HIVE_C0RE_NATS_URL == natsUrl
        && s.environment.HIVE_C0RE_OIDC_CLIENT_SECRET_FILE == "%d/swarm-status-client.secret"
        && lib.elem "swarm-status-client.secret:/var/lib/secrets/hive-h1.secret" (
          lib.toList s.serviceConfig.LoadCredential
        );
    }
    {
      # What the assertion was written for is still refused: a secret with
      # nowhere to present it.
      name = "a status secret without a queue URL is refused at eval";
      ok = refusedStatusSecret remoteSecretNoUrl && !(refusedStatusSecret remote);
    }
  ];
in
runGroup "nats-tls" cases