hyperhive/nix/module-eval/hive-tls.nix

# `checks.module-eval-hive-tls` — see ./lib.nix for the shared rationale (why
# this suite exists, naming convention, "evaluates not executes").
#
# The swarm-services leaf's renewal: a timer that really re-runs the
# issuance, at a threshold read off the store role's lifetime. And the
# store's root on a hive that asks it for no leaf. And the gateway leaf,
# issued by the store through the hive's own role, with no hive CA and no
# host-held swarm root.
{
  pkgs,
  lib,
  self,
  nixosSystem,
  shared,
}:
let
  inherit
    (import ./lib.nix {
      inherit
        pkgs
        lib
        self
        nixosSystem
        ;
    })
    hive
    hiveWithController
    runGroup
    ;

  # Every service on one host, so the store's granting unit renders the role
  # this leaf is issued through.
  inherit (shared) allLocal;

  # The same host with the role's lifetime moved, so a threshold that is a
  # number of its own shows up as one that did not move with it.
  shortTtl = hiveWithController {
    deploy.singleHostSwarm = true;
    deploy.bao.servicesPkiLeafTtlHours = 48;
  };

  # Fronting none of the swarm's names, reading a store it does not run,
  # with the reader identity, the services-issuer pair and the hive's own
  # identity placed by hand.
  noService = hiveWithController {
    deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
    deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
    deploy.bao.serverCaFile = "/etc/pki/bao-ca.pem";
    deploy.bao.servicesIssuerClientCertFile = "/etc/pki/bao-services-issuer.pem";
    deploy.bao.servicesIssuerClientKeyFile = "/etc/pki/bao-services-issuer-key.pem";
    deploy.hive-controller.tls.baoHiveIdentityCertFile = "/etc/pki/bao-hive.pem";
    deploy.hive-controller.tls.baoHiveIdentityKeyFile = "/etc/pki/bao-hive-key.pem";
  };
  noSvcFetch = noService.systemd.services.swarm-services-cert.script;

  # The same hive holding its own identity and no services-issuer pair: the
  # shape of every hive host without the store that fronts no swarm name.
  noServiceNoPair = hiveWithController {
    deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
    deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
    deploy.bao.serverCaFile = "/etc/pki/bao-ca.pem";
    deploy.hive-controller.tls.baoHiveIdentityCertFile = "/etc/pki/bao-hive.pem";
    deploy.hive-controller.tls.baoHiveIdentityKeyFile = "/etc/pki/bao-hive-key.pem";
  };
  noPairFetch = noServiceNoPair.systemd.services.swarm-services-cert.script;

  # The same hive with no store identity at all.
  noIdentity = shared.bareWithController;
  noIdFetch = noIdentity.systemd.services.swarm-services-cert.script;

  # A gateway on a host with no hive and no store, fronting no swarm name,
  # with the services-issuer pair placed by hand; and the same host without it.
  gatewayNoHive = hive {
    gateway.enable = true;
    deploy.bao.serverCaFile = "/etc/pki/bao-ca.pem";
    deploy.bao.servicesIssuerClientCertFile = "/etc/pki/bao-services-issuer.pem";
    deploy.bao.servicesIssuerClientKeyFile = "/etc/pki/bao-services-issuer-key.pem";
  };
  gatewayNoHiveNoPair = hive { gateway.enable = true; };

  # The store's host with the pair still named under its old path.
  renamedPair = hiveWithController {
    deploy.singleHostSwarm = true;
    deploy.hive-controller.tls.baoClientCertFile = "/etc/pki/old-issuer.pem";
    deploy.hive-controller.tls.baoClientKeyFile = "/etc/pki/old-issuer-key.pem";
  };

  # The `want_svc=0` branch of a rendered script, up to the `fi` that closes it (the
  # only one followed by a blank line).
  noSvcBranch =
    script:
    builtins.head (
      lib.splitString "\nfi\n\n" (lib.last (lib.splitString ''if [ "$want_svc" = 0 ]; then'' script))
    );

  # Everything after that branch: the path that logs in and issues the leaf.
  issuePath =
    script:
    lib.concatStringsSep "\nfi\n\n" (
      lib.tail (
        lib.splitString "\nfi\n\n" (lib.last (lib.splitString ''if [ "$want_svc" = 0 ]; then'' script))
      )
    );

  units = allLocal.systemd.services;
  boot = units.swarm-services-cert;
  caUnit = units.hive-tls-ca;
  leafUnit = units.hive-tls-resign;
  timer = allLocal.systemd.timers.swarm-services-cert-renew;

  # What the timer starts, read off the timer rather than assumed from its
  # name: a timer's `Unit=` defaults to its own name, and pointing it at the
  # boot unit is exactly the regression the cases below exist for.
  triggeredName = lib.removeSuffix ".service" (
    timer.timerConfig.Unit or "swarm-services-cert-renew.service"
  );
  triggered = units.${triggeredName};

  remainsAfterExit = u: u.serviceConfig.RemainAfterExit or false;

  # Whether `m` fails evaluation on `deploy.<option>` naming a deleted CA
  # file, with the message pointing at the store's mTLS CA.
  caFileRefused =
    m: option:
    lib.any (
      a:
      !a.assertion
      && lib.hasInfix "services.hyperhive.deploy.${option} is" a.message
      && lib.hasInfix "/var/lib/swarm-bao-pki/ca.pem" a.message
    ) m.assertions;
  caFileOptions = [
    "bao.clientCaFile"
    "bao.serverCaFile"
    "swarm-controller.hiveClientCaFile"
  ];

  renewAt = hours: "renewat=$(( ${toString hours} * 3600 / 2 ))";

  cases = [
    {
      # Control: the boot issuance is the unit a timer cannot re-run. Were it
      # not `RemainAfterExit`, the case after next would pass vacuously.
      name = "the boot issuance renders, and stays active after it exits";
      ok = lib.hasInfix "pki/issue/swarm-services" boot.script && remainsAfterExit boot;
    }
    {
      name = "a timer for the services leaf is enabled and fires daily";
      ok =
        lib.elem "timers.target" timer.wantedBy
        && timer.timerConfig.OnCalendar == "daily"
        && timer.timerConfig.Persistent;
    }
    {
      # Starting an active unit is a no-op, so a timer aimed at a
      # `RemainAfterExit` unit fires on schedule and renews nothing.
      name = "the timer starts a unit that re-runs the issuance and does not stay active";
      ok =
        units ? ${triggeredName}
        && !(remainsAfterExit triggered)
        && triggered.script == boot.script
        # The whole set, `PATH` included: an environment copied off the
        # boot unit's merged options renders a second `PATH` and fails.
        && triggered.environment == boot.environment;
    }
    {
      # A restart of anything the gateway's cert import `Requires=` takes
      # nginx down with it, so the renewal must be something nothing else
      # depends on, and must run behind the boot issuance.
      name = "nothing requires or waits on the renewal, and it runs after the boot issuance";
      ok =
        (triggered.requiredBy or [ ]) == [ ]
        && (triggered.wantedBy or [ ]) == [ ]
        && (triggered.before or [ ]) == [ ]
        && lib.elem "swarm-services-cert.service" triggered.after
        && !(lib.any (u: lib.elem "${triggeredName}.service" ((u.requires or [ ]) ++ (u.after or [ ]))) (
          lib.attrValues (removeAttrs units [ triggeredName ])
        ));
    }
    {
      # Moved with the role, in both places: the threshold is half of what the
      # store grants, and the store grants what the option says.
      name = "the leaf is renewed at half the role's lifetime, read from the option";
      ok =
        lib.hasInfix (renewAt 720) boot.script
        && lib.hasInfix (renewAt 48) shortTtl.systemd.services.swarm-services-cert.script
        && lib.hasInfix ''-in "$svcleaf" -noout -checkend "$renewat"'' boot.script
        && lib.hasInfix "max_ttl=48h" shortTtl.systemd.services.swarm-bao-controller-policy.script;
    }
    {
      # Checked on the decision side of the exit, in the boot issuance and
      # the renewal both.
      name = "the services leaf is re-issued when it does not chain to the store's root";
      ok =
        let
          verify = ''openssl verify -CAfile "$svcroot" "$svcleaf" >/dev/null 2>&1 || reissue=1'';
          decides =
            u:
            let
              decision = lib.head (lib.splitString ''if [ "$reissue" = 0 ]'' u.script);
            in
            decision != u.script && lib.hasInfix verify decision;
        in
        decides boot && decides triggered;
    }
    {
      # The store replaces its root once it has less than one leaf lifetime
      # left, and the hive asks for a fresh leaf, with the new root, at that
      # same threshold. A number of its own here keeps a leaf whose root the
      # store has already replaced.
      name = "the services root is re-checked at the store's own replacement threshold";
      ok =
        let
          shortBoot = shortTtl.systemd.services.swarm-services-cert.script;
          shortStore = shortTtl.systemd.services.swarm-bao-controller-policy.script;
        in
        lib.hasInfix ''-in "$svcroot" -noout -checkend ${toString (720 * 3600)} '' boot.script
        && lib.hasInfix ''-in "$svcroot" -noout -checkend ${toString (48 * 3600)} '' shortBoot
        && lib.hasInfix "-checkend ${toString (48 * 3600)} <<<\"$root_ca\"" shortStore;
    }
    {
      # The slot the bundle reads the store's root from, filled on a hive
      # that asks the store for no leaf.
      name = "a hive fronting no swarm name puts the store's pki root in its trust bundle";
      ok =
        noService.services.hyperhive.swarm.localServiceDomains == [ ]
        && lib.hasInfix ''"$BAO_ADDR/v1/pki/ca/pem"'' noSvcFetch
        && lib.hasInfix ''mv -f "$svcroot.new" "$svcroot"'' noSvcFetch
        && lib.hasInfix ''svcroot="$d/swarm-services-root.pem"'' noService.systemd.services.hive-tls-ca.script
        && lib.hasInfix ''cat "$svcroot" > "$bundle"'' noService.systemd.services.hive-tls-ca.script;
    }
    {
      # The hive's own identity, not its reader identity, whose policy reads
      # every secret the hive is entitled to, and not the services-issuer
      # pair, which `noService` also holds. A hive holding no pair fetches
      # the same way.
      name = "the root fetch reaches the store with the hive's own identity and server CA, never the services-issuer pair";
      ok =
        let
          hiveTls =
            "--cert ${lib.escapeShellArg "/etc/pki/bao-hive.pem"}"
            + " --key ${lib.escapeShellArg "/etc/pki/bao-hive-key.pem"}";
        in
        lib.hasInfix "--cacert ${lib.escapeShellArg "/etc/pki/bao-ca.pem"}" noSvcFetch
        && lib.hasInfix hiveTls (noSvcBranch noSvcFetch)
        && !(lib.hasInfix "/etc/pki/bao-client.pem" (noSvcBranch noSvcFetch))
        && !(lib.hasInfix "/etc/pki/bao-services-issuer" (noSvcBranch noSvcFetch))
        && noServiceNoPair.services.hyperhive.deploy.bao.servicesIssuerClientCertFile == null
        && lib.hasInfix hiveTls (noSvcBranch noPairFetch)
        && lib.hasInfix "/v1/pki/ca/pem" (noSvcBranch noPairFetch)
        && !(lib.hasInfix "servicesIssuerClient" (noSvcBranch noPairFetch));
    }
    {
      # No identity to present: the unit fails, naming both options, rather
      # than leaving the bundle without the root behind a green unit. The
      # control is the same hive with an identity, whose branch fetches.
      name = "a hive fronting no swarm name with no store identity fails the unit, naming the hive identity's options";
      ok =
        noIdentity.services.hyperhive.swarm.localServiceDomains == [ ]
        && noIdentity.services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile == null
        && !(lib.hasInfix "/v1/pki/ca/pem" noIdFetch)
        && lib.hasInfix "baoHiveIdentityCertFile (unset)" (noSvcBranch noIdFetch)
        && lib.hasInfix "baoHiveIdentityKeyFile (unset)" (noSvcBranch noIdFetch)
        && !(lib.hasInfix "servicesIssuerClient" (noSvcBranch noIdFetch))
        && lib.hasSuffix "exit 1" (lib.removeSuffix "\n" (noSvcBranch noIdFetch))
        && !(lib.hasInfix "exit 0" (noSvcBranch noIdFetch))
        && !(lib.hasInfix "not staged" noIdFetch)
        && lib.hasInfix "/v1/pki/ca/pem" (noSvcBranch noSvcFetch);
    }
    {
      # An identity named but not on disk fails before curl, naming the paths.
      name = "a hive fronting no swarm name whose identity files are absent fails the unit, naming the paths";
      ok =
        lib.hasInfix "for f in /etc/pki/bao-hive.pem /etc/pki/bao-hive-key.pem; do" (noSvcBranch noSvcFetch)
        && lib.hasInfix ''[ -r "$f" ] || missing="$missing $f"'' (noSvcBranch noSvcFetch)
        && lib.hasInfix "cannot read$missing" (noSvcBranch noSvcFetch)
        && lib.hasInfix "baoHiveIdentityCertFile (/etc/pki/bao-hive.pem)" (noSvcBranch noSvcFetch)
        && lib.hasInfix "baoHiveIdentityKeyFile (/etc/pki/bao-hive-key.pem)" (noSvcBranch noSvcFetch);
    }
    {
      # Each path presents one identity: the issue path the services-issuer
      # pair, on a hive host and on a host without a hive alike, and the root
      # fetch the hive's own. The control for the scan is `hive-tls-resign`,
      # which does present the hive identity.
      name = "the issue path presents only the services-issuer pair, and the root fetch only the hive identity";
      ok =
        let
          hiveRefs = [
            "baoHiveIdentity"
            "hive-leaf"
            "/etc/pki/bao-hive"
          ];
          refsHive = s: lib.any (r: lib.hasInfix r s) hiveRefs;
          issuesWithPair =
            m:
            let
              u = m.systemd.services;
              path = issuePath u.swarm-services-cert.script;
            in
            lib.hasInfix "for f in /etc/pki/bao-services-issuer.pem /etc/pki/bao-services-issuer-key.pem; do" path
            && !(refsHive path)
            &&
              lib.all
                (
                  e:
                  e.BAO_CLIENT_CERT == "/etc/pki/bao-services-issuer.pem"
                  && e.BAO_CLIENT_KEY == "/etc/pki/bao-services-issuer-key.pem"
                )
                [
                  u.swarm-services-cert.environment
                  u.swarm-services-cert-renew.environment
                ];
        in
        issuesWithPair noService
        && issuesWithPair gatewayNoHive
        && !gatewayNoHive.services.hyperhive.deploy.hive-controller.enable
        && refsHive (noSvcBranch noSvcFetch)
        && !(lib.hasInfix "/etc/pki/bao-services-issuer" (noSvcBranch noSvcFetch))
        && refsHive noService.systemd.services.hive-tls-resign.script;
    }
    {
      # A host with no store and no pair placed is a supported shape: it
      # evaluates, and its issue path goes red. A hive among them still
      # fetches the root. The controls are the same host without a hive with
      # the pair placed, and the store's host, where the glue supplies it.
      name = "a self-signed gateway with no services-issuer pair evaluates, and its issue path fails naming both options";
      ok =
        let
          refused =
            m:
            lib.any (
              a:
              !a.assertion
              && (
                lib.hasInfix "servicesIssuerClientCertFile" a.message
                || lib.hasInfix "servicesIssuerClientKeyFile" a.message
              )
            ) m.assertions;
          redIssue =
            m:
            lib.hasInfix (
              "set services.hyperhive.deploy.bao.servicesIssuerClientCertFile (unset)"
              + " and services.hyperhive.deploy.bao.servicesIssuerClientKeyFile (unset)"
            ) (issuePath m.systemd.services.swarm-services-cert.script);
        in
        !(refused gatewayNoHiveNoPair)
        && !(refused noIdentity)
        && !(refused noServiceNoPair)
        && redIssue gatewayNoHiveNoPair
        && redIssue noIdentity
        && redIssue noServiceNoPair
        && !(redIssue gatewayNoHive)
        && lib.hasInfix "/v1/pki/ca/pem" (noSvcBranch noPairFetch)
        &&
          allLocal.services.hyperhive.deploy.bao.servicesIssuerClientCertFile
          == "/var/lib/swarm-bao-pki/services-issuer.pem";
    }
    {
      # The old path's value reaches the new option over the store glue's
      # `mkDefault`, and the unit presents it. The control is the same store
      # host on its defaults.
      name = "the services-issuer pair set under deploy.hive-controller.tls reaches deploy.bao and the unit";
      ok =
        let
          b = renamedPair.services.hyperhive.deploy.bao;
          env = renamedPair.systemd.services.swarm-services-cert.environment;
        in
        b.servicesIssuerClientCertFile == "/etc/pki/old-issuer.pem"
        && b.servicesIssuerClientKeyFile == "/etc/pki/old-issuer-key.pem"
        && env.BAO_CLIENT_CERT == "/etc/pki/old-issuer.pem"
        && env.BAO_CLIENT_KEY == "/etc/pki/old-issuer-key.pem"
        &&
          allLocal.systemd.services.swarm-services-cert.environment.BAO_CLIENT_CERT
          == "/var/lib/swarm-bao-pki/services-issuer.pem";
    }
    {
      # The fetch retries against a sealed store; a gateway `Requires=` on it
      # would go down with each attempt. The control is a host fronting
      # swarm names, whose gateway does wait for its leaf.
      name = "a hive fronting no swarm name does not hold its gateway on the store";
      ok =
        let
          holds =
            m:
            lib.elem "hive-gateway-self-signed-cert.service" m.systemd.services.swarm-services-cert.requiredBy
            || lib.elem "hive-gateway-self-signed-cert.service" m.systemd.services.swarm-services-cert.before;
        in
        !(holds noService) && holds allLocal;
    }
    {
      # The control is the store's PKI unit, which still mints its own CA
      # with `openssl req`: absence here is not a matcher that cannot match.
      name = "no hive TLS unit generates a CA or signs with a CA key";
      ok =
        let
          hasCaWork =
            s:
            lib.any (p: lib.hasInfix p s) [
              "openssl req"
              "-CAkey"
            ];
          pkiScript = units.swarm-bao-pki.script;
        in
        !(hasCaWork caUnit.script)
        && !(hasCaWork leafUnit.script)
        && !(hasCaWork noService.systemd.services.hive-tls-ca.script)
        && lib.hasInfix "openssl req" pkiScript;
    }
    {
      # Every unit's script, environment and ordering, on the all-local host
      # and on a hive that runs no store. The control is the same scan finding
      # the store root's file, so an empty result is not a scan that reads
      # nothing.
      name = "no unit generates, reads or waits on a host-held swarm root, and every host deletes its directory";
      ok =
        let
          texts =
            m:
            lib.concatMap (u: [
              (u.script or "")
              (builtins.toJSON (u.environment or { }))
              (toString ((u.after or [ ]) ++ (u.requires or [ ]) ++ (u.wants or [ ])))
            ]) (lib.attrValues m.systemd.services);
          mentions = m: needle: lib.any (lib.hasInfix needle) (texts m);
          gone =
            m:
            !(m.systemd.services ? swarm-ca)
            && !(mentions m "swarm-ca.service")
            && !(mentions m "/var/lib/swarm-ca")
            && !(mentions m "root-key.pem")
            && lib.elem "R /var/lib/swarm-ca" m.systemd.tmpfiles.rules;
        in
        gone allLocal
        && gone noService
        && mentions allLocal "swarm-services-root.pem"
        && mentions noService "swarm-services-root.pem";
    }
    {
      # One anchor: any other slot is a root nothing issues under.
      name = "the trust bundle is the store's root alone";
      ok =
        lib.hasInfix ''cat "$svcroot" > "$bundle"'' caUnit.script
        && !(lib.hasInfix "anchors+=(" caUnit.script)
        && !(lib.hasInfix "ca.pem" caUnit.script);
    }
    {
      # A reload does not propagate through `Requires=`; a stop and start
      # restarts the store's container, which comes back sealed. The control
      # is that container still requiring the unit, so a failed assembly
      # still keeps it from starting.
      name = "a switch that changes the trust bundle's unit reloads it with the same script, so the store's container is not restarted";
      ok =
        let
          text = allLocal.systemd.units."hive-tls-ca.service".text;
          store = units."container@swarm-bao";
        in
        caUnit.reloadIfChanged
        && caUnit.reload == caUnit.script
        && lib.hasInfix "X-ReloadIfChanged=true" text
        && lib.hasInfix "ExecReload=" text
        && lib.elem "hive-tls-ca.service" store.requires
        && lib.elem "hive-tls-ca.service" store.after;
    }
    {
      # A missing root is the one case where the bundle is empty, and every
      # `<name>-ca-bundle` unit fails on it; the log names the file.
      name = "an empty trust bundle is logged naming the store's root it is copied from";
      ok = lib.hasInfix ''echo "the store's root $svcroot is missing or empty'' (
        lib.last (lib.splitString ''if [ -s "$svcroot" ]; then'' caUnit.script)
      );
    }
    {
      # The control is the all-local host, which defines none of them.
      name = "a definition of a removed swarm.ca option fails evaluation, naming the option and the store's root";
      ok =
        let
          refused =
            m: opt:
            lib.any (
              a:
              !a.assertion
              && lib.hasInfix "services.hyperhive.swarm.ca.${opt}" a.message
              && lib.hasInfix "pki root is the" a.message
            ) m.assertions;
          setting = {
            autoConfigure = hive { swarm.ca.autoConfigure = true; };
            stateDir = hive { swarm.ca.stateDir = "/var/lib/swarm-ca"; };
            validityDays = hive { swarm.ca.validityDays = 10950; };
          };
        in
        lib.all (opt: refused setting.${opt} opt) (lib.attrNames setting)
        && !(lib.any (opt: refused allLocal opt) (lib.attrNames setting));
    }
    {
      # Where the import `Requires=` the services leaf's issuing unit, a
      # missing leaf fails it; elsewhere the hive leaf stands in. The control
      # is a hive fronting no swarm name, which does not wait.
      name = "a hive gateway that waits for the services leaf fails its cert import without it, naming the issuing unit";
      ok =
        let
          imp = m: m.systemd.services.hive-gateway-self-signed-cert.script;
          waits =
            m:
            lib.elem "hive-gateway-self-signed-cert.service" m.systemd.services.swarm-services-cert.requiredBy;
          standIn = "install -m 0644 /var/lib/hive-tls/gateway.pem /var/lib/hive-gateway/tls/swarm-services.pem";
          refusal = "swarm-services-cert.service issues it";
          # The `else` branch alone, up to the `fi` that closes it.
          missingBranch =
            m:
            builtins.head (
              lib.splitString "\nfi" (lib.last (lib.splitString "swarm-services.pem ]; then" (imp m)))
            );
        in
        waits allLocal
        && lib.hasInfix refusal (missingBranch allLocal)
        && lib.hasSuffix "exit 1" (lib.removeSuffix "\n" (lib.trim (missingBranch allLocal)))
        && !(lib.hasInfix standIn (imp allLocal))
        && !(waits noService)
        && lib.hasInfix standIn (missingBranch noService)
        && !(lib.hasInfix refusal (imp noService));
    }
    {
      # The control is the same scan finding the store root's file, and a
      # relocated state dir, whose rules follow it.
      name = "no hive TLS unit reads a hive CA's files, and every host deletes them";
      ok =
        let
          texts =
            m:
            map (u: m.systemd.services.${u}.script) [
              "hive-tls-ca"
              "hive-tls-resign"
              "hive-gateway-self-signed-cert"
            ];
          reads = m: needle: lib.any (lib.hasInfix needle) (texts m);
          deletes =
            m: dir:
            lib.all (f: lib.elem "r ${dir}/${f}" m.systemd.tmpfiles.rules) [
              "ca.pem"
              "ca-previous.pem"
              "ca-key.pem"
              "ca.srl"
              ".swarm-ca-adopted"
            ];
          relocated = hiveWithController { deploy.hive-controller.tls.stateDir = "/srv/hive-tls"; };
        in
        !(lib.any (reads allLocal) [
          "/ca.pem"
          "ca-previous.pem"
          "ca-key.pem"
          "ca.srl"
          ".swarm-ca-adopted"
        ])
        && reads allLocal "swarm-services-root.pem"
        && deletes allLocal "/var/lib/hive-tls"
        && deletes noService "/var/lib/hive-tls"
        && deletes relocated "/srv/hive-tls"
        && !(deletes relocated "/var/lib/hive-tls");
    }
    {
      # The store host pointing its client CA at the old root: the store's and
      # the controller's options both fire, the second through the glue's
      # `mkDefault`. The control is the same host on its defaults.
      name = "a CA-file option naming a deleted CA file fails evaluation, naming the store's mTLS CA";
      ok =
        let
          oldRoot = hiveWithController {
            deploy.singleHostSwarm = true;
            deploy.bao.clientCaFile = "/var/lib/swarm-ca/root.pem";
          };
          hiveCa = hive {
            deploy.bao.serverCaFile = "/var/lib/hive-tls/ca.pem";
            deploy.swarm-controller.hiveClientCaFile = "/var/lib/hive-tls/ca-previous.pem";
          };
        in
        caFileRefused oldRoot "bao.clientCaFile"
        && caFileRefused oldRoot "swarm-controller.hiveClientCaFile"
        && caFileRefused hiveCa "bao.serverCaFile"
        && caFileRefused hiveCa "swarm-controller.hiveClientCaFile"
        && !(lib.any (caFileRefused allLocal) caFileOptions)
        && allLocal.services.hyperhive.deploy.bao.clientCaFile == "/var/lib/swarm-bao-pki/ca.pem";
    }
    {
      # The control is a host whose options normalise to paths beside the
      # deleted ones: a sibling of the swarm root's directory, and the
      # store's mTLS CA reached through `..`.
      name = "a CA-file option spelling a deleted path with `//`, `.`, `..` or a trailing slash fails evaluation too";
      ok =
        let
          spelled = hive {
            deploy.bao.clientCaFile = "/var/lib//swarm-ca/";
            deploy.bao.serverCaFile = "/var/lib/hive-tls/./ca.pem";
            deploy.swarm-controller.hiveClientCaFile = "/var/lib/hive-tls/x/../ca-key.pem";
          };
          beside = hive {
            deploy.bao.clientCaFile = "/var/lib//swarm-ca-copy/./root.pem";
            deploy.bao.serverCaFile = "/var/lib/hive-tls/../swarm-bao-pki/ca.pem";
            deploy.swarm-controller.hiveClientCaFile = "/var/lib/hive-tls/ca.pem/../trust-bundle.pem";
          };
        in
        lib.all (caFileRefused spelled) caFileOptions && !(lib.any (caFileRefused beside) caFileOptions);
    }
    {
      # The store host's hive presents the leaf ./glue-bao-tls.nix mints for
      # it; not the reader's, not the services issuer's.
      name = "the gateway leaf is issued through this hive's own role, presenting its own identity";
      ok =
        let
          t = allLocal.services.hyperhive.deploy.hive-controller.tls;
        in
        lib.hasInfix "pki/issue/swarm-hive-leaf-h1" leafUnit.script
        && lib.hasInfix "common_name=h1.t.local" leafUnit.script
        && lib.hasInfix "alt_names='*.h1.t.local,h1.hives.t.local'" leafUnit.script
        && lib.hasInfix "bao login -method=cert -token-only" leafUnit.script
        && leafUnit.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/hive-leaf.pem"
        && leafUnit.environment.BAO_CLIENT_KEY == "/var/lib/swarm-bao-pki/hive-leaf-key.pem"
        && t.baoHiveIdentityCertFile == leafUnit.environment.BAO_CLIENT_CERT
        &&
          leafUnit.environment.BAO_CLIENT_CERT
          != allLocal.services.hyperhive.deploy.bao.servicesIssuerClientCertFile
        && leafUnit.environment.BAO_CLIENT_CERT != allLocal.services.hyperhive.deploy.bao.clientCertFile;
    }
    {
      # Checked before the leaf, so the unit is red even while a valid leaf
      # keeps the gateway up. The control is a hive with the identity named,
      # whose script checks the files instead.
      name = "a hive with no store identity fails the gateway-leaf unit, naming both options";
      ok =
        let
          s = noIdentity.systemd.services.hive-tls-resign.script;
          before = lib.head (lib.splitString "reissue=0" s);
          ok' = noService.systemd.services.hive-tls-resign.script;
        in
        lib.hasInfix "services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile (unset)" before
        && lib.hasInfix "services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityKeyFile (unset)" before
        && lib.hasInfix "exit 1" before
        && lib.hasInfix "for f in /etc/pki/bao-hive.pem /etc/pki/bao-hive-key.pem; do" (
          lib.head (lib.splitString "reissue=0" ok')
        )
        && !(lib.hasInfix "(unset)" ok');
    }
    {
      # A cert from one issuance beside the key of another, left by an
      # interrupted pair of renames, is re-issued rather than left for
      # `nginx -t` to refuse. Checked on the decision side of the exit.
      name = "the gateway leaf is re-issued when not matching its key, not chained to the store's root, or at half its role lifetime";
      ok =
        let
          decision = lib.head (lib.splitString ''if [ "$reissue" = 0 ]'' leafUnit.script);
        in
        decision != leafUnit.script
        && lib.hasInfix ''openssl verify -CAfile "$svcroot" "$leaf"'' decision
        && lib.hasInfix ''leafpub="$(openssl x509 -in "$leaf" -noout -pubkey'' decision
        && lib.hasInfix ''keypub="$(openssl pkey -in "$leafk" -pubout'' decision
        && lib.hasInfix ''{ [ -n "$leafpub" ] && [ "$leafpub" = "$keypub" ]; } || reissue=1'' decision
        && lib.hasInfix (renewAt 720) leafUnit.script
        && lib.hasInfix (renewAt 48) shortTtl.systemd.services.hive-tls-resign.script
        && lib.hasInfix ''-in "$leaf" -noout -checkend "$renewat"'' leafUnit.script;
    }
    {
      # A valid leaf on disk starts the gateway with the store sealed: nothing
      # requires or waits on the issuing unit. The control is that unit
      # itself, which does wait on the store's container.
      name = "the gateway does not wait on the gateway-leaf unit, which waits on the store";
      ok =
        (leafUnit.requiredBy or [ ]) == [ ]
        && (leafUnit.before or [ ]) == [ ]
        && !(lib.any (u: lib.elem "hive-tls-resign.service" ((u.requires or [ ]) ++ (u.after or [ ]))) (
          lib.attrValues (removeAttrs units [ "hive-tls-resign" ])
        ))
        && !(lib.elem "container@swarm-bao.service" ((caUnit.requires or [ ]) ++ (caUnit.after or [ ])))
        && lib.elem "container@swarm-bao.service" leafUnit.after
        && lib.elem "multi-user.target" leafUnit.wantedBy;
    }
    {
      # Not `RemainAfterExit`, or the timer would start an active unit and
      # renew nothing.
      name = "a daily timer re-runs the gateway-leaf unit";
      ok =
        let
          t = allLocal.systemd.timers.hive-tls-resign;
        in
        lib.elem "timers.target" t.wantedBy
        && t.timerConfig.OnCalendar == "daily"
        && (t.timerConfig.Unit or "hive-tls-resign.service") == "hive-tls-resign.service"
        && !(remainsAfterExit leafUnit);
    }
    {
      # A oneshot has no start timeout by default, so an attempt that never
      # ends never fails into a retry.
      name = "every unit that calls the store bounds its attempt, and the root fetch bounds its curl";
      ok =
        lib.all (u: (u.serviceConfig.TimeoutStartSec or "infinity") != "infinity") [
          boot
          triggered
          leafUnit
        ]
        && lib.hasInfix "--connect-timeout 10 --max-time 30" (noSvcBranch noSvcFetch);
    }
  ];
in
runGroup "hive-tls" cases