# `checks.module-eval-hive-tls` — see ./lib.nix for the shared rationale (why # this suite exists, naming convention, "evaluates not executes"). # # The swarm-services leaf's renewal: a timer that really re-runs the # issuance, at a threshold read off the store role's lifetime. And the # store's root on a hive that asks it for no leaf. And the gateway leaf, # issued by the store through the hive's own role, with no hive CA and no # host-held swarm root. { pkgs, lib, self, nixosSystem, }: let inherit (import ./lib.nix { inherit pkgs lib self nixosSystem ; }) hive hiveWithController runGroup ; # Every service on one host, so the store's granting unit renders the role # this leaf is issued through. allLocal = hiveWithController { deploy.singleHostSwarm = true; }; # The same host with the role's lifetime moved, so a threshold that is a # number of its own shows up as one that did not move with it. shortTtl = hiveWithController { deploy.singleHostSwarm = true; deploy.bao.servicesPkiLeafTtlHours = 48; }; # Fronting none of the swarm's names, reading a store it does not run, # with the reader identity and the hive's own identity placed by hand. noService = hiveWithController { deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; deploy.bao.serverCaFile = "/etc/pki/bao-ca.pem"; deploy.hive-controller.tls.baoHiveIdentityCertFile = "/etc/pki/bao-hive.pem"; deploy.hive-controller.tls.baoHiveIdentityKeyFile = "/etc/pki/bao-hive-key.pem"; }; noSvcFetch = noService.systemd.services.swarm-services-cert.script; # The same hive with no store identity at all. noIdentity = hiveWithController { }; noIdFetch = noIdentity.systemd.services.swarm-services-cert.script; # The `want_svc=0` branch of a rendered script, up to the `fi` that closes it (the # only one followed by a blank line). noSvcBranch = script: builtins.head ( lib.splitString "\nfi\n\n" (lib.last (lib.splitString ''if [ "$want_svc" = 0 ]; then'' script)) ); units = allLocal.systemd.services; boot = units.swarm-services-cert; caUnit = units.hive-tls-ca; leafUnit = units.hive-tls-resign; timer = allLocal.systemd.timers.swarm-services-cert-renew; # What the timer starts, read off the timer rather than assumed from its # name: a timer's `Unit=` defaults to its own name, and pointing it at the # boot unit is exactly the regression the cases below exist for. triggeredName = lib.removeSuffix ".service" ( timer.timerConfig.Unit or "swarm-services-cert-renew.service" ); triggered = units.${triggeredName}; remainsAfterExit = u: u.serviceConfig.RemainAfterExit or false; renewAt = hours: "renewat=$(( ${toString hours} * 3600 / 2 ))"; cases = [ { # Control: the boot issuance is the unit a timer cannot re-run. Were it # not `RemainAfterExit`, the case after next would pass vacuously. name = "the boot issuance renders, and stays active after it exits"; ok = lib.hasInfix "pki/issue/swarm-services" boot.script && remainsAfterExit boot; } { name = "a timer for the services leaf is enabled and fires daily"; ok = lib.elem "timers.target" timer.wantedBy && timer.timerConfig.OnCalendar == "daily" && timer.timerConfig.Persistent; } { # Starting an active unit is a no-op, so a timer aimed at a # `RemainAfterExit` unit fires on schedule and renews nothing. name = "the timer starts a unit that re-runs the issuance and does not stay active"; ok = units ? ${triggeredName} && !(remainsAfterExit triggered) && triggered.script == boot.script # The whole set, `PATH` included: an environment copied off the # boot unit's merged options renders a second `PATH` and fails. && triggered.environment == boot.environment; } { # A restart of anything the gateway's cert import `Requires=` takes # nginx down with it, so the renewal must be something nothing else # depends on, and must run behind the boot issuance. name = "nothing requires or waits on the renewal, and it runs after the boot issuance"; ok = (triggered.requiredBy or [ ]) == [ ] && (triggered.wantedBy or [ ]) == [ ] && (triggered.before or [ ]) == [ ] && lib.elem "swarm-services-cert.service" triggered.after && !(lib.any (u: lib.elem "${triggeredName}.service" ((u.requires or [ ]) ++ (u.after or [ ]))) ( lib.attrValues (removeAttrs units [ triggeredName ]) )); } { # Moved with the role, in both places: the threshold is half of what the # store grants, and the store grants what the option says. name = "the leaf is renewed at half the role's lifetime, read from the option"; ok = lib.hasInfix (renewAt 720) boot.script && lib.hasInfix (renewAt 48) shortTtl.systemd.services.swarm-services-cert.script && lib.hasInfix ''-in "$svcleaf" -noout -checkend "$renewat"'' boot.script && lib.hasInfix "max_ttl=48h" shortTtl.systemd.services.swarm-bao-controller-policy.script; } { # The store replaces its root once it has less than one leaf lifetime # left, and the hive asks for a fresh leaf, with the new root, at that # same threshold. A number of its own here keeps a leaf whose root the # store has already replaced. name = "the services root is re-checked at the store's own replacement threshold"; ok = let shortBoot = shortTtl.systemd.services.swarm-services-cert.script; shortStore = shortTtl.systemd.services.swarm-bao-controller-policy.script; in lib.hasInfix ''-in "$svcroot" -noout -checkend ${toString (720 * 3600)} '' boot.script && lib.hasInfix ''-in "$svcroot" -noout -checkend ${toString (48 * 3600)} '' shortBoot && lib.hasInfix "-checkend ${toString (48 * 3600)} <<<\"$root_ca\"" shortStore; } { # The slot the bundle reads the store's root from, filled on a hive # that asks the store for no leaf. name = "a hive fronting no swarm name puts the store's pki root in its trust bundle"; ok = noService.services.hyperhive.swarm.localServiceDomains == [ ] && lib.hasInfix ''"$BAO_ADDR/v1/pki/ca/pem"'' noSvcFetch && lib.hasInfix ''mv -f "$svcroot.new" "$svcroot"'' noSvcFetch && lib.hasInfix ''svcroot="$d/swarm-services-root.pem"'' noService.systemd.services.hive-tls-ca.script && lib.hasInfix ''cat "$svcroot" > "$bundle"'' noService.systemd.services.hive-tls-ca.script; } { # The hive's own identity, not its reader identity: the reader's policy # reads every secret the hive is entitled to. name = "the root fetch reaches the store with the hive's own identity and server CA"; ok = lib.hasInfix "--cacert ${lib.escapeShellArg "/etc/pki/bao-ca.pem"}" noSvcFetch && lib.hasInfix ( "--cert ${lib.escapeShellArg "/etc/pki/bao-hive.pem"}" + " --key ${lib.escapeShellArg "/etc/pki/bao-hive-key.pem"}" ) noSvcFetch && !(lib.hasInfix "/etc/pki/bao-client.pem" (noSvcBranch noSvcFetch)); } { # No identity to present: the unit fails, naming both options, rather # than leaving the bundle without the root behind a green unit. The # control is the same hive with an identity, whose branch fetches. name = "a hive fronting no swarm name with no store identity fails the unit, naming both options"; ok = noIdentity.services.hyperhive.swarm.localServiceDomains == [ ] && noIdentity.services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile == null && !(lib.hasInfix "/v1/pki/ca/pem" noIdFetch) && lib.hasInfix "baoHiveIdentityCertFile (unset)" (noSvcBranch noIdFetch) && lib.hasInfix "baoHiveIdentityKeyFile (unset)" (noSvcBranch noIdFetch) && lib.hasSuffix "exit 1" (lib.removeSuffix "\n" (noSvcBranch noIdFetch)) && !(lib.hasInfix "exit 0" (noSvcBranch noIdFetch)) && !(lib.hasInfix "not staged" noIdFetch) && lib.hasInfix "/v1/pki/ca/pem" (noSvcBranch noSvcFetch); } { # An identity named but not on disk fails before curl, naming the paths. name = "a hive fronting no swarm name whose identity files are absent fails the unit, naming the paths"; ok = lib.hasInfix "for f in /etc/pki/bao-hive.pem /etc/pki/bao-hive-key.pem; do" (noSvcBranch noSvcFetch) && lib.hasInfix ''[ -r "$f" ] || missing="$missing $f"'' (noSvcBranch noSvcFetch) && lib.hasInfix "cannot read$missing" (noSvcBranch noSvcFetch) && lib.hasInfix "baoHiveIdentityCertFile (/etc/pki/bao-hive.pem)" (noSvcBranch noSvcFetch) && lib.hasInfix "baoHiveIdentityKeyFile (/etc/pki/bao-hive-key.pem)" (noSvcBranch noSvcFetch); } { # The fetch retries against a sealed store; a gateway `Requires=` on it # would go down with each attempt. The control is a host fronting # swarm names, whose gateway does wait for its leaf. name = "a hive fronting no swarm name does not hold its gateway on the store"; ok = let holds = m: lib.elem "hive-gateway-self-signed-cert.service" m.systemd.services.swarm-services-cert.requiredBy || lib.elem "hive-gateway-self-signed-cert.service" m.systemd.services.swarm-services-cert.before; in !(holds noService) && holds allLocal; } { # The control is the store's PKI unit, which still mints its own CA # with `openssl req`: absence here is not a matcher that cannot match. name = "no hive TLS unit generates a CA or signs with a CA key"; ok = let hasCaWork = s: lib.any (p: lib.hasInfix p s) [ "openssl req" "-CAkey" ]; pkiScript = units.swarm-bao-pki.script; in !(hasCaWork caUnit.script) && !(hasCaWork leafUnit.script) && !(hasCaWork noService.systemd.services.hive-tls-ca.script) && lib.hasInfix "openssl req" pkiScript; } { # Every unit's script, environment and ordering, on the all-local host # and on a hive that runs no store. The control is the same scan finding # the store root's file, so an empty result is not a scan that reads # nothing. name = "no unit generates, reads or waits on a host-held swarm root, and every host deletes its directory"; ok = let texts = m: lib.concatMap (u: [ (u.script or "") (builtins.toJSON (u.environment or { })) (toString ((u.after or [ ]) ++ (u.requires or [ ]) ++ (u.wants or [ ]))) ]) (lib.attrValues m.systemd.services); mentions = m: needle: lib.any (lib.hasInfix needle) (texts m); gone = m: !(m.systemd.services ? swarm-ca) && !(mentions m "swarm-ca.service") && !(mentions m "/var/lib/swarm-ca") && !(mentions m "root-key.pem") && lib.elem "R /var/lib/swarm-ca" m.systemd.tmpfiles.rules; in gone allLocal && gone noService && mentions allLocal "swarm-services-root.pem" && mentions noService "swarm-services-root.pem"; } { # One anchor: any other slot is a root nothing issues under. name = "the trust bundle is the store's root alone"; ok = lib.hasInfix ''cat "$svcroot" > "$bundle"'' caUnit.script && !(lib.hasInfix "anchors+=(" caUnit.script) && !(lib.hasInfix "ca.pem" caUnit.script); } { # The control is the all-local host, which defines none of them. name = "a definition of a removed swarm.ca option fails evaluation, naming the option and the store's root"; ok = let refused = m: opt: lib.any ( a: !a.assertion && lib.hasInfix "services.hyperhive.swarm.ca.${opt}" a.message && lib.hasInfix "pki root is the" a.message ) m.assertions; setting = { autoConfigure = hive { swarm.ca.autoConfigure = true; }; stateDir = hive { swarm.ca.stateDir = "/var/lib/swarm-ca"; }; validityDays = hive { swarm.ca.validityDays = 10950; }; }; in lib.all (opt: refused setting.${opt} opt) (lib.attrNames setting) && !(lib.any (opt: refused allLocal opt) (lib.attrNames setting)); } { # Where the import `Requires=` the services leaf's issuing unit, a # missing leaf fails it; elsewhere the hive leaf stands in. The control # is a hive fronting no swarm name, which does not wait. name = "a hive gateway that waits for the services leaf fails its cert import without it, naming the issuing unit"; ok = let imp = m: m.systemd.services.hive-gateway-self-signed-cert.script; waits = m: lib.elem "hive-gateway-self-signed-cert.service" m.systemd.services.swarm-services-cert.requiredBy; standIn = "install -m 0644 /var/lib/hive-tls/gateway.pem /var/lib/hive-gateway/tls/swarm-services.pem"; refusal = "swarm-services-cert.service issues it"; # The `else` branch alone, up to the `fi` that closes it. missingBranch = m: builtins.head ( lib.splitString "\nfi" (lib.last (lib.splitString "swarm-services.pem ]; then" (imp m))) ); in waits allLocal && lib.hasInfix refusal (missingBranch allLocal) && lib.hasSuffix "exit 1" (lib.removeSuffix "\n" (lib.trim (missingBranch allLocal))) && !(lib.hasInfix standIn (imp allLocal)) && !(waits noService) && lib.hasInfix standIn (missingBranch noService) && !(lib.hasInfix refusal (imp noService)); } { # The control is the same scan finding the store root's file, and a # relocated state dir, whose rules follow it. name = "no hive TLS unit reads a hive CA's files, and every host deletes them"; ok = let texts = m: map (u: m.systemd.services.${u}.script) [ "hive-tls-ca" "hive-tls-resign" "hive-gateway-self-signed-cert" ]; reads = m: needle: lib.any (lib.hasInfix needle) (texts m); deletes = m: dir: lib.all (f: lib.elem "r ${dir}/${f}" m.systemd.tmpfiles.rules) [ "ca.pem" "ca-previous.pem" "ca-key.pem" ]; relocated = hiveWithController { deploy.hive-controller.tls.stateDir = "/srv/hive-tls"; }; in !(lib.any (reads allLocal) [ "/ca.pem" "ca-previous.pem" "ca-key.pem" ]) && reads allLocal "swarm-services-root.pem" && deletes allLocal "/var/lib/hive-tls" && deletes noService "/var/lib/hive-tls" && deletes relocated "/srv/hive-tls" && !(deletes relocated "/var/lib/hive-tls"); } { # The store host pointing its client CA at the old root: the store's and # the controller's options both fire, the second through the glue's # `mkDefault`. The control is the same host on its defaults. name = "a CA-file option naming a deleted CA file fails evaluation, naming the store's mTLS CA"; ok = let refused = m: option: lib.any ( a: !a.assertion && lib.hasInfix "services.hyperhive.deploy.${option} is" a.message && lib.hasInfix "/var/lib/swarm-bao-pki/ca.pem" a.message ) m.assertions; oldRoot = hiveWithController { deploy.singleHostSwarm = true; deploy.bao.clientCaFile = "/var/lib/swarm-ca/root.pem"; }; hiveCa = hive { deploy.bao.serverCaFile = "/var/lib/hive-tls/ca.pem"; deploy.swarm-controller.hiveClientCaFile = "/var/lib/hive-tls/ca-previous.pem"; }; options = [ "bao.clientCaFile" "bao.serverCaFile" "swarm-controller.hiveClientCaFile" ]; in refused oldRoot "bao.clientCaFile" && refused oldRoot "swarm-controller.hiveClientCaFile" && refused hiveCa "bao.serverCaFile" && refused hiveCa "swarm-controller.hiveClientCaFile" && !(lib.any (refused allLocal) options) && allLocal.services.hyperhive.deploy.bao.clientCaFile == "/var/lib/swarm-bao-pki/ca.pem"; } { # The store host's hive presents the leaf ./glue-bao-tls.nix mints for # it; not the reader's, not the services issuer's. name = "the gateway leaf is issued through this hive's own role, presenting its own identity"; ok = let t = allLocal.services.hyperhive.deploy.hive-controller.tls; in lib.hasInfix "pki/issue/swarm-hive-leaf-h1" leafUnit.script && lib.hasInfix "common_name=h1.t.local" leafUnit.script && lib.hasInfix "alt_names='*.h1.t.local'" leafUnit.script && lib.hasInfix "bao login -method=cert -token-only" leafUnit.script && leafUnit.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/hive-leaf.pem" && leafUnit.environment.BAO_CLIENT_KEY == "/var/lib/swarm-bao-pki/hive-leaf-key.pem" && t.baoHiveIdentityCertFile == leafUnit.environment.BAO_CLIENT_CERT && leafUnit.environment.BAO_CLIENT_CERT != t.baoClientCertFile && leafUnit.environment.BAO_CLIENT_CERT != allLocal.services.hyperhive.deploy.bao.clientCertFile; } { # Checked before the leaf, so the unit is red even while a valid leaf # keeps the gateway up. The control is a hive with the identity named, # whose script checks the files instead. name = "a hive with no store identity fails the gateway-leaf unit, naming both options"; ok = let s = noIdentity.systemd.services.hive-tls-resign.script; before = lib.head (lib.splitString "reissue=0" s); ok' = noService.systemd.services.hive-tls-resign.script; in lib.hasInfix "services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile (unset)" before && lib.hasInfix "services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityKeyFile (unset)" before && lib.hasInfix "exit 1" before && lib.hasInfix "for f in /etc/pki/bao-hive.pem /etc/pki/bao-hive-key.pem; do" ( lib.head (lib.splitString "reissue=0" ok') ) && !(lib.hasInfix "(unset)" ok'); } { # A cert from one issuance beside the key of another, left by an # interrupted pair of renames, is re-issued rather than left for # `nginx -t` to refuse. Checked on the decision side of the exit. name = "the gateway leaf is re-issued when not matching its key, not chained to the store's root, or at half its role lifetime"; ok = let decision = lib.head (lib.splitString ''if [ "$reissue" = 0 ]'' leafUnit.script); in decision != leafUnit.script && lib.hasInfix ''openssl verify -CAfile "$svcroot" "$leaf"'' decision && lib.hasInfix ''leafpub="$(openssl x509 -in "$leaf" -noout -pubkey'' decision && lib.hasInfix ''keypub="$(openssl pkey -in "$leafk" -pubout'' decision && lib.hasInfix ''{ [ -n "$leafpub" ] && [ "$leafpub" = "$keypub" ]; } || reissue=1'' decision && lib.hasInfix (renewAt 720) leafUnit.script && lib.hasInfix (renewAt 48) shortTtl.systemd.services.hive-tls-resign.script && lib.hasInfix ''-in "$leaf" -noout -checkend "$renewat"'' leafUnit.script; } { # A valid leaf on disk starts the gateway with the store sealed: nothing # requires or waits on the issuing unit. The control is that unit # itself, which does wait on the store's container. name = "the gateway does not wait on the gateway-leaf unit, which waits on the store"; ok = (leafUnit.requiredBy or [ ]) == [ ] && (leafUnit.before or [ ]) == [ ] && !(lib.any (u: lib.elem "hive-tls-resign.service" ((u.requires or [ ]) ++ (u.after or [ ]))) ( lib.attrValues (removeAttrs units [ "hive-tls-resign" ]) )) && !(lib.elem "container@swarm-bao.service" ((caUnit.requires or [ ]) ++ (caUnit.after or [ ]))) && lib.elem "container@swarm-bao.service" leafUnit.after && lib.elem "multi-user.target" leafUnit.wantedBy; } { # Not `RemainAfterExit`, or the timer would start an active unit and # renew nothing. name = "a daily timer re-runs the gateway-leaf unit"; ok = let t = allLocal.systemd.timers.hive-tls-resign; in lib.elem "timers.target" t.wantedBy && t.timerConfig.OnCalendar == "daily" && (t.timerConfig.Unit or "hive-tls-resign.service") == "hive-tls-resign.service" && !(remainsAfterExit leafUnit); } ]; in runGroup "hive-tls" cases