hyperhive/nix/module-eval/bao-basics.nix

# `checks.module-eval-bao-basics` — see ./lib.nix for the shared
# rationale (why this suite exists, naming convention, "evaluates
# not executes").
{
  pkgs,
  lib,
  self,
  nixosSystem,
}:
let
  inherit
    (import ./lib.nix {
      inherit
        pkgs
        lib
        self
        nixosSystem
        ;
    })
    hive
    runGroup
    baoNames
    baoSettings
    baoStream
    bridgePorts
    ;

  baoPkcs11 = hive {
    deploy.bao.enable = true;
    deploy.bao.seal = "pkcs11";
  };

  baoShamir = hive {
    deploy.bao.enable = true;
    deploy.bao.seal = "shamir";
  };

  baoExplicitCerts = hive {
    deploy.bao.enable = true;
    deploy.bao.serverCertFile = "/etc/pki/bao.pem";
    deploy.bao.serverKeyFile = "/etc/pki/bao-key.pem";
  };

  # The store's units live inside its container, so the gates below have to
  # look there rather than at the host's service set.
  baoUnits = machine: machine.containers.swarm-bao.config.systemd.services;

  tlsDir = "/var/lib/swarm-bao-tls";
  cases = [
    {
      # The store's seal is spread over six gates — the stanza, the
      # provisioning unit, two bind mounts, a device and an EnvironmentFile.
      # Rendering only some of them is the dangerous state: a store that
      # says hardware-backed and seals with a software key, which no
      # assertion can catch because every value is individually valid.
      name = "a shamir store renders no TPM provisioning unit";
      ok = !(baoUnits baoShamir ? swarm-bao-token);
    }
    {
      # Presence control for the case above. Without it, a typo in the
      # option name would satisfy the absence arm forever. The second half is
      # the fix itself: the unit has to run where openbao's `DynamicUser` is
      # allocated, and a host unit writing the same bytes has no name to hand
      # them to.
      name = "a pkcs11 store provisions the token in the container, not on the host";
      ok = (baoUnits baoPkcs11 ? swarm-bao-token) && !(baoPkcs11.systemd.services ? swarm-bao-token);
    }
    {
      # `allowedDevices` renders `DeviceAllow=` and nothing else — nspawn
      # mounts its own /dev and cannot create device nodes, so permission to
      # use a device that was never bound in opens nothing. Neither half
      # fails on its own, which is why they are asserted as a pair.
      name = "a pkcs11 store gets the TPM device bound in, not merely allowed";
      ok =
        let
          c = baoPkcs11.containers.swarm-bao;
        in
        (c.bindMounts ? "/dev/tpmrm0") && builtins.any (d: d.node == "/dev/tpmrm0") c.allowedDevices;
    }
    {
      # The stanza's label and the label the unit creates are two literals that
      # have to name one object, and the mechanism is asserted with them
      # because it is valid only for an RSA key: openbao takes AES-GCM or
      # RSA-OAEP and a TPM 2.0 has neither GCM nor an opinion about which the
      # seal asked for. Every wrong combination renders and deploys, and
      # surfaces as a pkcs11 error at `operator init`.
      name = "the seal asks for the RSA key the provisioning unit creates";
      ok =
        let
          p = (baoSettings baoPkcs11).seal.pkcs11;
        in
        (p.mechanism or "") == "CKM_RSA_PKCS_OAEP"
        && lib.hasInfix "--algorithm=rsa2048 --key-label=${p.key_label or ""}" (
          (baoUnits baoPkcs11).swarm-bao-token.script or ""
        );
    }
    {
      # `DynamicUser` implies `ProtectSystem=strict`, so the token directory is
      # read-only to the seal however it is owned, and the group is the only
      # handle on a uid allocated at start. Dropping either surfaces as a
      # pkcs11 error deep in a library, naming neither the mount nor the user.
      name = "the store's seal may write the token directory, and is in both its groups";
      ok =
        let
          sc = (baoUnits baoPkcs11).openbao.serviceConfig;
        in
        # `or [ ]` rather than a bare select: the interesting mutation is the
        # key being gone, and a select would abort the whole run with a nix
        # trace instead of failing this case by name.
        builtins.elem "/var/lib/swarm-bao-token" (sc.ReadWritePaths or [ ])
        && builtins.elem "swarm-bao-token" (sc.SupplementaryGroups or [ ])
        && builtins.elem "swarm-bao-tpm" (sc.SupplementaryGroups or [ ]);
    }
    {
      # The device node belongs to the HOST and is matched by NUMBER, while the
      # unit that opens it lives in the container — so the two sides holding
      # the same gid is the entire mechanism. Letting either side auto-allocate
      # renders cleanly, deploys cleanly, and leaves a 0660 node the seal
      # cannot open. Compared rather than each checked against a literal: the
      # property is that they AGREE, not what they agree on.
      name = "the TPM group has the same gid on the host and inside the container";
      ok =
        let
          host = baoPkcs11.users.groups.swarm-bao-tpm.gid or null;
          inner = baoPkcs11.containers.swarm-bao.config.users.groups.swarm-bao-tpm.gid or null;
        in
        host != null && host == inner;
    }
    {
      # Absence arm for the case above — a shamir store never opens a TPM, so
      # it must not claim a device node's group. Without this, pinning the gid
      # unconditionally would look identical.
      name = "a shamir store claims no TPM device group";
      ok = !(baoShamir.users.groups ? swarm-bao-tpm);
    }
    {
      # The store's mTLS identity is a separate trust domain from both CAs in
      # this tree, because it must not come from an authority the store will
      # itself distribute. What supplies it is the glue, which mints a CA of
      # the store's own — so an enabled store has all three paths, and if this
      # ever reads null again the store stops coming up on its own.
      name = "a deployed store is given its own certificate, key and client CA";
      ok =
        let
          b = baoPkcs11.services.hyperhive.deploy.bao;
        in
        b.serverCertFile != null && b.serverKeyFile != null && b.clientCaFile != null;
    }
    {
      # Everything the glue sets is `mkDefault`, and this is the case that
      # says so: a deployment whose certificates come from somewhere the glue
      # has never heard of must win. Also the presence control for the case
      # above — a renamed option would read `null` on both and satisfy
      # neither, but only this one names a value.
      name = "an operator's own certificate path beats the glue's default";
      ok = baoExplicitCerts.services.hyperhive.deploy.bao.serverCertFile == "/etc/pki/bao.pem";
    }
    {
      # Absence arm. A store with nothing to serve renders no reader, so the
      # unit is a function of the PAIRING rather than of the store — which is
      # the property that makes it glue instead of a feature of either side.
      name = "a store with no homeserver beside it renders no token reader";
      ok = !(baoPkcs11.systemd.services ? swarm-bao-matrix-token);
    }
    {
      # The name a reader dials has to resolve where the store runs; a
      # multi-host swarm resolves it upstream instead.
      name = "the store's host answers for the store's name";
      ok = builtins.elem "bao.t.local" (baoNames baoPkcs11);
    }
    {
      # What makes that name reachable from inside an agent container, and the
      # single reason it is a stream server rather than a vhost: `ssl_preread`
      # routes on the SNI without decrypting, so the handshake bao completes is
      # still the client's own and the certificate it authenticates by arrives
      # intact. Terminating here would hand the store one identity for the
      # whole swarm.
      name = "the store's host passes connections through without terminating TLS";
      ok =
        let
          s = baoStream baoPkcs11;
        in
        lib.hasInfix "ssl_preread on;" s && lib.hasInfix "proxy_pass $swarm_bao_backend;" s;
    }
    {
      # The address half, and it is bridge-only for a reason a wildcard would
      # hide until deploy: the store already holds `127.0.0.1:8200` in this
      # same netns, so `0.0.0.0:8200` is `EADDRINUSE` and nginx fails to start
      # — taking every hive domain behind the gateway down with it.
      name = "the passthrough listens on the bridge, not on every address";
      ok = lib.hasInfix "listen 10.42.0.1:8200;" (baoStream baoPkcs11);
    }
    {
      # The routing half: the SNI picks the backend and the only name that
      # resolves to one is the store's own. A `default` that pointed anywhere
      # would make this host a relay for whatever name a client invented.
      name = "the passthrough routes only the store's name, to its loopback listener";
      ok =
        let
          s = baoStream baoPkcs11;
        in
        lib.hasInfix "map $ssl_preread_server_name $swarm_bao_backend" s
        && lib.hasInfix "bao.t.local 127.0.0.1:8200;" s
        && lib.hasInfix ''default "";'' s;
    }
    {
      # The listener is only half of reachable: the bridge firewall drops
      # everything not named here, and a silent drop is the failure that reads
      # as "the store is down" from inside a container.
      name = "the store's port is open on the bridge where the store runs";
      ok = builtins.elem 8200 (bridgePorts baoPkcs11);
    }
    {
      # Raft refuses to start without it, and says so in a message that names
      # neither the setting nor the stanza.
      name = "the store advertises a cluster address";
      ok = lib.hasPrefix "https://" ((baoSettings baoPkcs11).cluster_addr or "");
    }
    {
      # The listener trusts the agent CA through a file no cert-auth role
      # names. At least one listener verifies clients, so an empty set cannot
      # pass.
      name = "every client-verifying listener reads the listener-only bundle";
      ok =
        let
          verifying = lib.filter (l: l ? tls_client_ca_file) (
            lib.attrValues (baoSettings baoPkcs11).listener
          );
        in
        verifying != [ ]
        && lib.all (l: l.tls_client_ca_file == "${tlsDir}/listener-client-ca.pem") verifying;
    }
    {
      # The other half of keeping agents out of host roles: those roles pin
      # the store CA alone. The positive count is the control that the text
      # searched is the one the roles are written in.
      name = "host cert-auth roles pin client-ca.pem and never the listener bundle or the agent CA";
      ok =
        let
          scripts = lib.concatStrings (
            lib.mapAttrsToList (
              n: u: lib.optionalString (lib.hasPrefix "swarm-bao-" n) (u.script or "")
            ) baoPkcs11.systemd.services
          );
        in
        lib.hasInfix "certificate=@${tlsDir}/client-ca.pem" scripts
        && !(lib.hasInfix "certificate=@${tlsDir}/listener-client-ca.pem" scripts)
        && !(lib.hasInfix "certificate=@${tlsDir}/agent-ca.pem" scripts);
    }
    {
      # Both writers compose the bundle through one function, which refuses
      # a result that does not begin with the store CA and replaces the file
      # only when its bytes change.
      name = "both bundle writers use the one composer, which keeps the store CA first";
      ok =
        let
          s = baoPkcs11.systemd.services;
          composes =
            u:
            lib.hasInfix "compose_listener_bundle() {" u.script
            && lib.hasInfix "\ncompose_listener_bundle\n" u.script
            && lib.hasInfix ''cmp -s -n "$(stat -c %s ${tlsDir}/client-ca.pem)" ${tlsDir}/client-ca.pem "$tmp"'' u.script
            && lib.hasInfix ''cmp -s "$tmp" ${tlsDir}/listener-client-ca.pem'' u.script
            && lib.hasInfix "mktemp -p ${tlsDir} " u.script;
        in
        s ? swarm-bao-agent-pki && composes s.swarm-bao-certs && composes s.swarm-bao-agent-pki;
    }
    {
      # openbao reads its client-CA file only at start, so picking up the
      # agent CA is a restart: automatic where the store unseals itself,
      # printed where a human has to. The shamir arm is the control.
      name = "the agent PKI unit restarts openbao under pkcs11 and only prints the step under shamir";
      ok =
        let
          restart = ''systemctl --machine="$machine" restart openbao.service'';
          p = baoPkcs11.systemd.services.swarm-bao-agent-pki.script;
          sh = baoShamir.systemd.services.swarm-bao-agent-pki.script;
        in
        lib.hasInfix restart p
        && lib.hasInfix ''if [ "$written_us" -le "$started_us" ]; then'' p
        && !(lib.hasInfix restart sh)
        && lib.hasInfix "# then unseal" sh;
    }
  ];
in
runGroup "bao-basics" cases