hyperhive/nix/module-eval/nats-tls.nix

# `checks.module-eval-nats-tls` — see ./lib.nix for the shared rationale (why
# this suite exists, naming convention, "evaluates not executes").
#
# The queue's name, its bao-issued leaf, and the clients that dial it.
{
  pkgs,
  lib,
  self,
  nixosSystem,
  shared,
}:
let
  inherit
    (import ./lib.nix {
      inherit
        pkgs
        lib
        self
        nixosSystem
        ;
    })
    hive
    hiveWithController
    runGroup
    bridgePorts
    ;

  natsName = "nats.t.local";
  natsUrl = "tls://${natsName}:4222";

  # Every service on one host. The queue, the store and every in-tree client of the queue
  # are all here, so the scan below reads each of them.
  inherit (shared) allLocal;

  # The same host on the mesh.
  allLocalMesh = hive {
    deploy.singleHostSwarm = true;
    deploy.wireguard.enable = true;
    deploy.wireguard.address = "10.100.0.1/24";
  };

  # The queue on a host whose store is elsewhere: no local policy unit.
  queueNoStore = hive {
    deploy.nats.enable = true;
  };

  # The store's host with the hive's own reader pointed somewhere else.
  natsOnStoreHostReaderMoved = hive {
    deploy.bao.enable = true;
    deploy.nats.enable = true;
    deploy.nats.autoGenerateCallout = true;
    deploy.bao.clientCertFile = "/etc/pki/h1.pem";
    deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem";
  };

  # A queue host holding only the hive's own store reader, placed by hand: no
  # store here, and no queue leaf named.
  queueWithHiveReaderOnly = hive {
    deploy.nats.enable = true;
    deploy.nats.autoGenerateCallout = true;
    deploy.bao.clientCertFile = "/etc/pki/h1.pem";
    deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem";
  };

  # A hive that is not the queue's host, with nothing about the queue's
  # address set by hand: what every hive but one in a multi-host swarm looks
  # like. The controller is on too, since it may run away from the queue.
  #
  # The forge's secret and the controller's store identity are what a hive
  # away from authelia and the store already has to be handed, and none is an
  # address; without them this hive would fail assertions that have nothing to
  # do with the queue.
  remoteSecrets = {
    deploy.forgejo.sso.clientSecretFile = "/var/lib/forgejo-oidc/by-hand.secret";
    deploy.swarm-controller.baoClientCertFile = "/var/lib/swarm-controller/bao-client.pem";
    deploy.swarm-controller.baoClientKeyFile = "/var/lib/swarm-controller/bao-client-key.pem";
  };
  remote = hiveWithController (
    lib.recursiveUpdate remoteSecrets { deploy.swarm-controller.enable = true; }
  );

  # The same hive given its own store leaf by hand, which is what turns
  # publishing on away from the IdP's host: hive-c0re reads the status secret
  # from the store under it. The queue-credential reader's pair comes with it,
  # because a hive that reads the store is refused at eval without one.
  hiveLeaf = {
    deploy.bao.clientCertFile = "/etc/pki/h1.pem";
    deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem";
    deploy.bao.queueAgentClientCertFile = "/etc/pki/h1-queue-agent.pem";
    deploy.bao.queueAgentClientKeyFile = "/etc/pki/h1-queue-agent-key.pem";
  };
  remotePublishing = hiveWithController (lib.recursiveUpdate remoteSecrets hiveLeaf);

  # The same leaf with the queue's URL taken away.
  remoteLeafNoUrl = hiveWithController (
    lib.recursiveUpdate remoteSecrets (
      lib.recursiveUpdate hiveLeaf { deploy.hive-controller.statusPublish.natsUrl = null; }
    )
  );

  failedAssertions = m: lib.filter (a: !a.assertion) m.assertions;

  policyScript = allLocal.systemd.services.swarm-bao-nats-tls-policy.script;
  leafUnit = allLocal.systemd.services.swarm-bao-nats-tls;
  natsContainer = allLocal.containers.swarm-nats.config;
  natsTls = natsContainer.services.nats.settings.tls;

  # Every `(nats|tls)://…` in a string.
  urlsIn =
    s: map builtins.head (builtins.filter builtins.isList (builtins.split "((nats|tls)://[^ '\"]+)" s));

  # Every in-tree queue client, found rather than listed. The Rust client reads
  # its address from `<PREFIX>_NATS_URL` (`swarm_queue_client::QueueConfig::
  # from_env`), so any unit on the host or in a container that is handed one
  # carries a variable of that shape. The responder takes a flag instead.
  # Keyed by where each came from, so the control below can name them.
  clientUrls =
    machine:
    let
      fromUnits =
        where: services:
        lib.concatLists (
          lib.mapAttrsToList (
            unit: s:
            lib.mapAttrsToList (var: v: {
              name = "${where}/${unit}/${var}";
              value = v;
            }) (lib.filterAttrs (var: v: lib.hasSuffix "_NATS_URL" var && v != null) (s.environment or { }))
          ) services
        );
      containerUnits = lib.concatLists (
        lib.mapAttrsToList (c: cc: fromUnits c cc.config.systemd.services) machine.containers
      );
      # The responder runs beside the queue only, so a hive without one has
      # none to read.
      responder =
        map
          (u: {
            name = "swarm-nats/swarm-nats-auth/--nats-url";
            value = u;
          })
          (
            lib.optionals (machine.containers ? swarm-nats) (
              urlsIn machine.containers.swarm-nats.config.systemd.services.swarm-nats-auth.serviceConfig.ExecStart
            )
          );
    in
    lib.listToAttrs (fromUnits "host" machine.systemd.services ++ containerUnits ++ responder);

  scanned = clientUrls allLocal;

  cases = [
    {
      # Control first: a scan that found nothing would pass the next case
      # vacuously. These are the four clients in the tree today.
      name = "the client scan finds hive-c0re, the agents, the controller and the responder";
      ok = lib.all (k: scanned ? ${k}) [
        "host/hive-c0re/HIVE_C0RE_NATS_URL"
        "host/hive-c0re/HIVE_AGENT_NATS_URL"
        "host/swarm-controller/SWARM_CONTROLLER_NATS_URL"
        "swarm-nats/swarm-nats-auth/--nats-url"
      ];
    }
    {
      # The server requires TLS and its leaf carries the name alone, so a
      # `nats://` URL or an address is a client that cannot connect. Every one
      # found, not the four above: a client added later is held to it too.
      name = "every in-tree queue client dials tls://<the queue's name>:4222";
      ok = lib.all (u: u == natsUrl) (lib.attrValues scanned);
    }
    {
      # The option defaults the scan reads through, so a client that stops
      # reading them does not also escape the property above.
      name = "the queue URL options default to the name on the queue's host";
      ok =
        let
          d = allLocal.services.hyperhive.deploy;
        in
        d.hive-controller.statusPublish.natsUrl == natsUrl
        && d.hive-controller.queue.agentNatsUrl == natsUrl
        && allLocal.services.hyperhive.swarm.controller.queue.natsUrl == natsUrl;
    }
    {
      name = "the queue's name is served by this host's resolver, at the bridge address";
      ok =
        lib.elem natsName allLocal.services.hyperhive.gateway.localNames
        && lib.elem "/${natsName}/${allLocal.services.hyperhive.network.bridgeIp}" allLocal.services.dnsmasq.settings.address;
    }
    {
      name = "the queue's pki role issues for its name alone";
      ok = lib.all (arg: lib.hasInfix arg policyScript) [
        "roles/swarm-nats \\"
        "allowed_domains=${lib.escapeShellArg natsName} \\"
        "allow_bare_domains=true \\"
        "allow_subdomains=false \\"
        "allow_glob_domains=false \\"
        "allow_localhost=false \\"
        "allow_any_name=false \\"
        "allow_ip_sans=false \\"
        "server_flag=true \\"
        "client_flag=false \\"
      ];
    }
    {
      # Every `path` the policy names, not a search for the one expected: a
      # second grant added later fails here.
      name = "the queue's policy grants pki/issue/swarm-nats and nothing else";
      ok =
        let
          paths = map builtins.head (
            builtins.filter builtins.isList (builtins.split "path \"([^\"]*)\"" policyScript)
          );
        in
        paths == [ "pki/issue/swarm-nats" ]
        && lib.hasInfix ''capabilities = ["update"]'' policyScript
        && lib.hasInfix "allowed_common_names=swarm-nats" policyScript
        && lib.hasInfix "token_policies=swarm-nats" policyScript;
    }
    {
      # Mint to consume: the leaf the unit writes is the one the server reads,
      # and the login leaf glue-bao-tls signs is the one the unit presents.
      name = "the server serves the leaf the host unit issues, and the unit logs in as swarm-nats";
      ok =
        natsTls.cert_file == "/var/lib/swarm-nats-tls/cert.pem"
        && natsTls.key_file == "/run/credentials/nats.service/tls-key"
        && lib.elem "tls-key:/var/lib/swarm-nats-tls/key.pem" natsContainer.systemd.services.nats.serviceConfig.LoadCredential
        && allLocal.containers.swarm-nats.bindMounts ? "/var/lib/swarm-nats-tls"
        && lib.hasInfix "d=/var/lib/swarm-nats-tls" leafUnit.script
        && lib.hasInfix "pki/issue/swarm-nats" leafUnit.script
        && leafUnit.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/nats.pem"
        && lib.hasInfix "[ -s /var/lib/swarm-bao-pki/nats.pem ]" allLocal.systemd.services.swarm-bao-pki.script
        && lib.hasInfix "swarm-nats \"\" clientAuth" allLocal.systemd.services.swarm-bao-pki.script;
    }
    {
      # The hive's own reader is not the queue's identity, and no path is
      # derived from it: a queue host without the store that holds only the
      # hive's reader gets the leaf unit that fails naming the options, like
      # one holding nothing.
      name = "a queue host without the store is refused when it holds only the hive's reader";
      ok =
        let
          u = queueWithHiveReaderOnly.systemd.services.swarm-bao-nats-tls;
        in
        queueWithHiveReaderOnly.services.hyperhive.deploy.nats.baoClientCertFile == null
        && !(u.environment ? BAO_CLIENT_CERT)
        && lib.hasInfix "services.hyperhive.deploy.nats.baoClient{Cert,Key}File" u.script;
    }
    {
      # The default is the file the minting unit writes, wherever the hive's
      # own reader is pointed.
      name = "on the store's host the queue's leaf is the one minted there, not one beside the hive's reader";
      ok =
        let
          n = natsOnStoreHostReaderMoved.services.hyperhive.deploy.nats;
          mint = natsOnStoreHostReaderMoved.systemd.services.swarm-bao-pki.script;
        in
        n.baoClientCertFile == "/var/lib/swarm-bao-pki/nats.pem"
        && n.baoClientKeyFile == "/var/lib/swarm-bao-pki/nats-key.pem"
        && lib.hasInfix "[ -s /var/lib/swarm-bao-pki/nats.pem ] ||" mint
        && lib.hasInfix "/var/lib/swarm-bao-pki nats " mint;
    }
    {
      name = "the server requires TLS: no allow_non_tls";
      ok = !(natsContainer.services.nats.settings ? allow_non_tls);
    }
    {
      name = "4222 is open on wg-hive when this host is on the mesh, and never host-wide";
      ok =
        lib.elem 4222 allLocalMesh.networking.firewall.interfaces.wg-hive.allowedTCPPorts
        && !(lib.elem 4222 allLocalMesh.networking.firewall.allowedTCPPorts)
        && lib.elem 4222 (bridgePorts allLocalMesh);
    }
    {
      name = "4222 is not opened on wg-hive when this host is not on the mesh";
      ok =
        !(lib.elem 4222
          (allLocal.networking.firewall.interfaces.wg-hive or { allowedTCPPorts = [ ]; }).allowedTCPPorts
        )
        && !(lib.elem 4222 allLocal.networking.firewall.allowedTCPPorts);
    }
    {
      # Ordering, never a requirement: a policy unit that failed still counts
      # as done, and the leaf unit's own retries carry it past that.
      name = "the leaf unit is ordered after its policy unit, with no requires";
      ok =
        let
          p = "swarm-bao-nats-tls-policy.service";
        in
        lib.elem p leafUnit.after && lib.elem p leafUnit.wants && !(lib.elem p (leafUnit.requires or [ ]));
    }
    {
      name = "a queue host whose store is elsewhere orders its leaf unit after no policy unit";
      ok =
        let
          u = queueNoStore.systemd.services.swarm-bao-nats-tls;
        in
        !(lib.elem "swarm-bao-nats-tls-policy.service" u.after)
        && !(lib.elem "swarm-bao-nats-tls-policy.service" u.wants);
    }
    {
      # Every hive dials the queue by the same name, so a hive away from it
      # needs no URL of its own. Control and property in one: the scan must
      # reach the controller and the agents' address here, and every URL it
      # finds, like the options it reads through, is the name.
      name = "a hive that is not the queue's host dials tls://<the queue's name>:4222 with nothing set";
      ok =
        let
          s = clientUrls remote;
          d = remote.services.hyperhive.deploy;
        in
        !d.nats.enable
        && s ? "host/hive-c0re/HIVE_AGENT_NATS_URL"
        && s ? "host/swarm-controller/SWARM_CONTROLLER_NATS_URL"
        && lib.all (u: u == natsUrl) (lib.attrValues s)
        && d.hive-controller.statusPublish.natsUrl == natsUrl
        && d.hive-controller.queue.agentNatsUrl == natsUrl
        && remote.services.hyperhive.swarm.controller.queue.natsUrl == natsUrl;
    }
    {
      # The URL is set and the hive holds no store leaf, which is every such
      # hive until an operator places one: publishing is off, not
      # misconfigured.
      name = "that hive evaluates without an assertion failure";
      ok = failedAssertions remote == [ ];
    }
    {
      # Off means off: hive-c0re is handed no status coordinates it could not
      # fetch a secret for.
      name = "without a store leaf, that hive's hive-c0re is given no status coordinates";
      ok = !(remote.systemd.services.hive-c0re.environment ? HIVE_C0RE_NATS_URL);
    }
    {
      # The leaf alone turns publishing on, at the default URL, with the
      # secret read from the store rather than named as a file.
      name = "given its store leaf, that hive publishes to the queue's name as hive-<name>";
      ok =
        let
          s = remotePublishing.systemd.services.hive-c0re;
        in
        failedAssertions remotePublishing == [ ]
        && s.environment.HIVE_C0RE_NATS_URL == natsUrl
        && s.environment.HIVE_C0RE_OIDC_CLIENT_ID == "hive-h1"
        && !(s.environment ? HIVE_C0RE_OIDC_CLIENT_SECRET_FILE);
    }
    {
      # The host that runs the queue, the IdP and the store gets the same
      # coordinates from the leaf ../host-modules/glue-bao-tls.nix mints there,
      # and no file.
      name = "the all-local hive publishes from the store too";
      ok =
        let
          e = allLocal.systemd.services.hive-c0re.environment;
        in
        e.HIVE_C0RE_NATS_URL == natsUrl
        && e.HIVE_C0RE_OIDC_CLIENT_ID == "hive-h1"
        && !(e ? HIVE_C0RE_OIDC_CLIENT_SECRET_FILE);
    }
    {
      # A null URL turns publishing off.
      name = "a hive with a store leaf and no queue URL is given no status coordinates";
      ok =
        failedAssertions remoteLeafNoUrl == [ ]
        && !(remoteLeafNoUrl.systemd.services.hive-c0re.environment ? HIVE_C0RE_NATS_URL);
    }
    {
      # Only `/metrics`, from the exporter, behind the collector's subrequest:
      # no `error_page` (a scraper would parse a login page as metrics), the
      # monitoring endpoint never proxied, every other path refused. In
      # self-signed mode the vhost names the queue's own leaf, as copied for
      # nginx, not the swarm-services one.
      name = "the queue's name serves only /metrics, behind the collector's auth subrequest, on the queue's own leaf";
      ok =
        let
          h = allLocal.services.hyperhive;
          v = allLocal.services.nginx.virtualHosts.${natsName};
          locs = v.locations;
          checksOf = l: lib.filter (lib.hasPrefix "auth_request") (lib.splitString "\n" l.extraConfig);
          authz = locs."= /__nats_metrics_authz";
        in
        lib.attrNames locs == [
          "/"
          "= /__nats_metrics_authz"
          "= /metrics"
        ]
        && locs."/".return == "403"
        && locs."/".proxyPass == null
        && locs."= /metrics".proxyPass == "http://127.0.0.1:${toString h.swarm.nats.metricsPort}/metrics"
        && checksOf locs."= /metrics" == [ "auth_request /__nats_metrics_authz;" ]
        && !(lib.hasInfix "error_page" locs."= /metrics".extraConfig)
        && authz.proxyPass == "https://auth.t.local/api/authz/auth-request"
        && lib.hasInfix "internal;" authz.extraConfig
        && !(lib.any (
          l:
          lib.hasInfix ":${toString h.swarm.nats.monitorPort}" (
            if l.proxyPass == null then "" else l.proxyPass
          )
        ) (lib.attrValues locs))
        && v.forceSSL
        && v.sslCertificate == "/var/lib/swarm-nats-gateway-tls/cert.pem"
        && v.sslCertificateKey == "/var/lib/swarm-nats-gateway-tls/key.pem";
    }
    {
      # A host whose only swarm service is the queue: its gateway serves the
      # queue's name with the queue's leaf, fronts no swarm-services name, and
      # so neither waits for nor requires the swarm-services leaf. The copy
      # unit is only wanted by nginx, never required, and a rotated queue leaf
      # re-runs it.
      name = "a host running only the queue renders its gateway without requiring the swarm-services leaf";
      ok =
        let
          h = queueNoStore.services.hyperhive;
          s = queueNoStore.systemd.services;
          copy = s.swarm-nats-gateway-cert;
        in
        h.deploy.nats.enable
        && !h.deploy.hive-controller.enable
        && h.gateway.enable
        && h.gateway.useSelfSigned
        &&
          queueNoStore.services.nginx.virtualHosts.${natsName}.sslCertificate
          == "/var/lib/swarm-nats-gateway-tls/cert.pem"
        && h.swarm.localServiceDomains == [ ]
        && !(lib.elem natsName h.swarm.serviceDomains)
        && !(lib.elem "hive-gateway-self-signed-cert.service" (s.swarm-services-cert.requiredBy or [ ]))
        && !(lib.hasInfix "serves swarm names only with it" s.hive-gateway-self-signed-cert.script)
        && lib.elem "nginx.service" copy.wantedBy
        && lib.elem "nginx.service" copy.before
        && !(lib.elem "nginx.service" (copy.requiredBy or [ ]))
        && lib.hasInfix "throwaway self-signed" copy.script
        && lib.hasInfix "chgrp nginx" copy.script
        && lib.hasInfix "systemctl start swarm-nats-gateway-cert.service" s.swarm-bao-nats-tls.script;
    }
  ];
in
runGroup "nats-tls" cases