Watch
0
0
Fork
You've already forked hyperhive
0
hyperhive/nix/module-eval/hive-tls.nix
atlas 4e5a642928 hive CA: delete ca.pem, ca-previous.pem and ca-key.pem, the bundle is the store's root
The gateway leaf comes from the store's pki mount, so a hive CA's
certificates anchor nothing and its key is a secret outside the store.

- hive-tls-ca writes trust-bundle.pem from swarm-services-root.pem alone;
  the ca.pem and ca-previous.pem slots are gone.
- Every host deletes ca.pem, ca-previous.pem and ca-key.pem in
  deploy.hive-controller.tls.stateDir with tmpfiles `r` rules that follow
  the option. The removal shim, now host-ca-removed.nix, holds these
  beside the /var/lib/swarm-ca rule and the swarm.ca.* removals.
- deploy.bao.clientCaFile, deploy.bao.serverCaFile and
  deploy.swarm-controller.hiveClientCaFile naming /var/lib/swarm-ca or one
  of the deleted hive CA files fail evaluation, naming the store's mTLS CA
  (/var/lib/swarm-bao-pki/ca.pem on the store's host) as the replacement.

module-eval hive-tls covers the single-anchor bundle, the deletion (incl.
a relocated stateDir) and the assertion on all three options.

Refs #4588
2026-10-05 21:17:10 +02:00

473 lines
21 KiB
Nix

# `checks.module-eval-hive-tls` — see ./lib.nix for the shared rationale (why
# this suite exists, naming convention, "evaluates not executes").
#
# The swarm-services leaf's renewal: a timer that really re-runs the
# issuance, at a threshold read off the store role's lifetime. And the
# store's root on a hive that asks it for no leaf. And the gateway leaf,
# issued by the store through the hive's own role, with no hive CA and no
# host-held swarm root.
{
pkgs,
lib,
self,
nixosSystem,
}:
let
inherit
(import ./lib.nix {
inherit
pkgs
lib
self
nixosSystem
;
})
hive
hiveWithController
runGroup
;
# Every service on one host, so the store's granting unit renders the role
# this leaf is issued through.
allLocal = hiveWithController {
deploy.singleHostSwarm = true;
};
# The same host with the role's lifetime moved, so a threshold that is a
# number of its own shows up as one that did not move with it.
shortTtl = hiveWithController {
deploy.singleHostSwarm = true;
deploy.bao.servicesPkiLeafTtlHours = 48;
};
# Fronting none of the swarm's names, reading a store it does not run,
# with the reader identity and the hive's own identity placed by hand.
noService = hiveWithController {
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
deploy.bao.serverCaFile = "/etc/pki/bao-ca.pem";
deploy.hive-controller.tls.baoHiveIdentityCertFile = "/etc/pki/bao-hive.pem";
deploy.hive-controller.tls.baoHiveIdentityKeyFile = "/etc/pki/bao-hive-key.pem";
};
noSvcFetch = noService.systemd.services.swarm-services-cert.script;
# The same hive with no store identity at all.
noIdentity = hiveWithController { };
noIdFetch = noIdentity.systemd.services.swarm-services-cert.script;
# The `want_svc=0` branch of a rendered script, up to the `fi` that closes it (the
# only one followed by a blank line).
noSvcBranch =
script:
builtins.head (
lib.splitString "\nfi\n\n" (lib.last (lib.splitString ''if [ "$want_svc" = 0 ]; then'' script))
);
units = allLocal.systemd.services;
boot = units.swarm-services-cert;
caUnit = units.hive-tls-ca;
leafUnit = units.hive-tls-resign;
timer = allLocal.systemd.timers.swarm-services-cert-renew;
# What the timer starts, read off the timer rather than assumed from its
# name: a timer's `Unit=` defaults to its own name, and pointing it at the
# boot unit is exactly the regression the cases below exist for.
triggeredName = lib.removeSuffix ".service" (
timer.timerConfig.Unit or "swarm-services-cert-renew.service"
);
triggered = units.${triggeredName};
remainsAfterExit = u: u.serviceConfig.RemainAfterExit or false;
renewAt = hours: "renewat=$(( ${toString hours} * 3600 / 2 ))";
cases = [
{
# Control: the boot issuance is the unit a timer cannot re-run. Were it
# not `RemainAfterExit`, the case after next would pass vacuously.
name = "the boot issuance renders, and stays active after it exits";
ok = lib.hasInfix "pki/issue/swarm-services" boot.script && remainsAfterExit boot;
}
{
name = "a timer for the services leaf is enabled and fires daily";
ok =
lib.elem "timers.target" timer.wantedBy
&& timer.timerConfig.OnCalendar == "daily"
&& timer.timerConfig.Persistent;
}
{
# Starting an active unit is a no-op, so a timer aimed at a
# `RemainAfterExit` unit fires on schedule and renews nothing.
name = "the timer starts a unit that re-runs the issuance and does not stay active";
ok =
units ? ${triggeredName}
&& !(remainsAfterExit triggered)
&& triggered.script == boot.script
# The whole set, `PATH` included: an environment copied off the
# boot unit's merged options renders a second `PATH` and fails.
&& triggered.environment == boot.environment;
}
{
# A restart of anything the gateway's cert import `Requires=` takes
# nginx down with it, so the renewal must be something nothing else
# depends on, and must run behind the boot issuance.
name = "nothing requires or waits on the renewal, and it runs after the boot issuance";
ok =
(triggered.requiredBy or [ ]) == [ ]
&& (triggered.wantedBy or [ ]) == [ ]
&& (triggered.before or [ ]) == [ ]
&& lib.elem "swarm-services-cert.service" triggered.after
&& !(lib.any (u: lib.elem "${triggeredName}.service" ((u.requires or [ ]) ++ (u.after or [ ]))) (
lib.attrValues (removeAttrs units [ triggeredName ])
));
}
{
# Moved with the role, in both places: the threshold is half of what the
# store grants, and the store grants what the option says.
name = "the leaf is renewed at half the role's lifetime, read from the option";
ok =
lib.hasInfix (renewAt 720) boot.script
&& lib.hasInfix (renewAt 48) shortTtl.systemd.services.swarm-services-cert.script
&& lib.hasInfix ''-in "$svcleaf" -noout -checkend "$renewat"'' boot.script
&& lib.hasInfix "max_ttl=48h" shortTtl.systemd.services.swarm-bao-controller-policy.script;
}
{
# The store replaces its root once it has less than one leaf lifetime
# left, and the hive asks for a fresh leaf, with the new root, at that
# same threshold. A number of its own here keeps a leaf whose root the
# store has already replaced.
name = "the services root is re-checked at the store's own replacement threshold";
ok =
let
shortBoot = shortTtl.systemd.services.swarm-services-cert.script;
shortStore = shortTtl.systemd.services.swarm-bao-controller-policy.script;
in
lib.hasInfix ''-in "$svcroot" -noout -checkend ${toString (720 * 3600)} '' boot.script
&& lib.hasInfix ''-in "$svcroot" -noout -checkend ${toString (48 * 3600)} '' shortBoot
&& lib.hasInfix "-checkend ${toString (48 * 3600)} <<<\"$root_ca\"" shortStore;
}
{
# The slot the bundle reads the store's root from, filled on a hive
# that asks the store for no leaf.
name = "a hive fronting no swarm name puts the store's pki root in its trust bundle";
ok =
noService.services.hyperhive.swarm.localServiceDomains == [ ]
&& lib.hasInfix ''"$BAO_ADDR/v1/pki/ca/pem"'' noSvcFetch
&& lib.hasInfix ''mv -f "$svcroot.new" "$svcroot"'' noSvcFetch
&& lib.hasInfix ''svcroot="$d/swarm-services-root.pem"'' noService.systemd.services.hive-tls-ca.script
&& lib.hasInfix ''cat "$svcroot" > "$bundle"'' noService.systemd.services.hive-tls-ca.script;
}
{
# The hive's own identity, not its reader identity: the reader's policy
# reads every secret the hive is entitled to.
name = "the root fetch reaches the store with the hive's own identity and server CA";
ok =
lib.hasInfix "--cacert ${lib.escapeShellArg "/etc/pki/bao-ca.pem"}" noSvcFetch
&& lib.hasInfix (
"--cert ${lib.escapeShellArg "/etc/pki/bao-hive.pem"}"
+ " --key ${lib.escapeShellArg "/etc/pki/bao-hive-key.pem"}"
) noSvcFetch
&& !(lib.hasInfix "/etc/pki/bao-client.pem" (noSvcBranch noSvcFetch));
}
{
# No identity to present: the unit fails, naming both options, rather
# than leaving the bundle without the root behind a green unit. The
# control is the same hive with an identity, whose branch fetches.
name = "a hive fronting no swarm name with no store identity fails the unit, naming both options";
ok =
noIdentity.services.hyperhive.swarm.localServiceDomains == [ ]
&& noIdentity.services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile == null
&& !(lib.hasInfix "/v1/pki/ca/pem" noIdFetch)
&& lib.hasInfix "baoHiveIdentityCertFile (unset)" (noSvcBranch noIdFetch)
&& lib.hasInfix "baoHiveIdentityKeyFile (unset)" (noSvcBranch noIdFetch)
&& lib.hasSuffix "exit 1" (lib.removeSuffix "\n" (noSvcBranch noIdFetch))
&& !(lib.hasInfix "exit 0" (noSvcBranch noIdFetch))
&& !(lib.hasInfix "not staged" noIdFetch)
&& lib.hasInfix "/v1/pki/ca/pem" (noSvcBranch noSvcFetch);
}
{
# An identity named but not on disk fails before curl, naming the paths.
name = "a hive fronting no swarm name whose identity files are absent fails the unit, naming the paths";
ok =
lib.hasInfix "for f in /etc/pki/bao-hive.pem /etc/pki/bao-hive-key.pem; do" (noSvcBranch noSvcFetch)
&& lib.hasInfix ''[ -r "$f" ] || missing="$missing $f"'' (noSvcBranch noSvcFetch)
&& lib.hasInfix "cannot read$missing" (noSvcBranch noSvcFetch)
&& lib.hasInfix "baoHiveIdentityCertFile (/etc/pki/bao-hive.pem)" (noSvcBranch noSvcFetch)
&& lib.hasInfix "baoHiveIdentityKeyFile (/etc/pki/bao-hive-key.pem)" (noSvcBranch noSvcFetch);
}
{
# The fetch retries against a sealed store; a gateway `Requires=` on it
# would go down with each attempt. The control is a host fronting
# swarm names, whose gateway does wait for its leaf.
name = "a hive fronting no swarm name does not hold its gateway on the store";
ok =
let
holds =
m:
lib.elem "hive-gateway-self-signed-cert.service" m.systemd.services.swarm-services-cert.requiredBy
|| lib.elem "hive-gateway-self-signed-cert.service" m.systemd.services.swarm-services-cert.before;
in
!(holds noService) && holds allLocal;
}
{
# The control is the store's PKI unit, which still mints its own CA
# with `openssl req`: absence here is not a matcher that cannot match.
name = "no hive TLS unit generates a CA or signs with a CA key";
ok =
let
hasCaWork =
s:
lib.any (p: lib.hasInfix p s) [
"openssl req"
"-CAkey"
];
pkiScript = units.swarm-bao-pki.script;
in
!(hasCaWork caUnit.script)
&& !(hasCaWork leafUnit.script)
&& !(hasCaWork noService.systemd.services.hive-tls-ca.script)
&& lib.hasInfix "openssl req" pkiScript;
}
{
# Every unit's script, environment and ordering, on the all-local host
# and on a hive that runs no store. The control is the same scan finding
# the store root's file, so an empty result is not a scan that reads
# nothing.
name = "no unit generates, reads or waits on a host-held swarm root, and every host deletes its directory";
ok =
let
texts =
m:
lib.concatMap (u: [
(u.script or "")
(builtins.toJSON (u.environment or { }))
(toString ((u.after or [ ]) ++ (u.requires or [ ]) ++ (u.wants or [ ])))
]) (lib.attrValues m.systemd.services);
mentions = m: needle: lib.any (lib.hasInfix needle) (texts m);
gone =
m:
!(m.systemd.services ? swarm-ca)
&& !(mentions m "swarm-ca.service")
&& !(mentions m "/var/lib/swarm-ca")
&& !(mentions m "root-key.pem")
&& lib.elem "R /var/lib/swarm-ca" m.systemd.tmpfiles.rules;
in
gone allLocal
&& gone noService
&& mentions allLocal "swarm-services-root.pem"
&& mentions noService "swarm-services-root.pem";
}
{
# One anchor: any other slot is a root nothing issues under.
name = "the trust bundle is the store's root alone";
ok =
lib.hasInfix ''cat "$svcroot" > "$bundle"'' caUnit.script
&& !(lib.hasInfix "anchors+=(" caUnit.script)
&& !(lib.hasInfix "ca.pem" caUnit.script);
}
{
# The control is the all-local host, which defines none of them.
name = "a definition of a removed swarm.ca option fails evaluation, naming the option and the store's root";
ok =
let
refused =
m: opt:
lib.any (
a:
!a.assertion
&& lib.hasInfix "services.hyperhive.swarm.ca.${opt}" a.message
&& lib.hasInfix "pki root is the" a.message
) m.assertions;
setting = {
autoConfigure = hive { swarm.ca.autoConfigure = true; };
stateDir = hive { swarm.ca.stateDir = "/var/lib/swarm-ca"; };
validityDays = hive { swarm.ca.validityDays = 10950; };
};
in
lib.all (opt: refused setting.${opt} opt) (lib.attrNames setting)
&& !(lib.any (opt: refused allLocal opt) (lib.attrNames setting));
}
{
# Where the import `Requires=` the services leaf's issuing unit, a
# missing leaf fails it; elsewhere the hive leaf stands in. The control
# is a hive fronting no swarm name, which does not wait.
name = "a hive gateway that waits for the services leaf fails its cert import without it, naming the issuing unit";
ok =
let
imp = m: m.systemd.services.hive-gateway-self-signed-cert.script;
waits =
m:
lib.elem "hive-gateway-self-signed-cert.service" m.systemd.services.swarm-services-cert.requiredBy;
standIn = "install -m 0644 /var/lib/hive-tls/gateway.pem /var/lib/hive-gateway/tls/swarm-services.pem";
refusal = "swarm-services-cert.service issues it";
# The `else` branch alone, up to the `fi` that closes it.
missingBranch =
m:
builtins.head (
lib.splitString "\nfi" (lib.last (lib.splitString "swarm-services.pem ]; then" (imp m)))
);
in
waits allLocal
&& lib.hasInfix refusal (missingBranch allLocal)
&& lib.hasSuffix "exit 1" (lib.removeSuffix "\n" (lib.trim (missingBranch allLocal)))
&& !(lib.hasInfix standIn (imp allLocal))
&& !(waits noService)
&& lib.hasInfix standIn (missingBranch noService)
&& !(lib.hasInfix refusal (imp noService));
}
{
# The control is the same scan finding the store root's file, and a
# relocated state dir, whose rules follow it.
name = "no hive TLS unit reads a hive CA's files, and every host deletes them";
ok =
let
texts =
m:
map (u: m.systemd.services.${u}.script) [
"hive-tls-ca"
"hive-tls-resign"
"hive-gateway-self-signed-cert"
];
reads = m: needle: lib.any (lib.hasInfix needle) (texts m);
deletes =
m: dir:
lib.all (f: lib.elem "r ${dir}/${f}" m.systemd.tmpfiles.rules) [
"ca.pem"
"ca-previous.pem"
"ca-key.pem"
];
relocated = hiveWithController { deploy.hive-controller.tls.stateDir = "/srv/hive-tls"; };
in
!(lib.any (reads allLocal) [
"/ca.pem"
"ca-previous.pem"
"ca-key.pem"
])
&& reads allLocal "swarm-services-root.pem"
&& deletes allLocal "/var/lib/hive-tls"
&& deletes noService "/var/lib/hive-tls"
&& deletes relocated "/srv/hive-tls"
&& !(deletes relocated "/var/lib/hive-tls");
}
{
# The store host pointing its client CA at the old root: the store's and
# the controller's options both fire, the second through the glue's
# `mkDefault`. The control is the same host on its defaults.
name = "a CA-file option naming a deleted CA file fails evaluation, naming the store's mTLS CA";
ok =
let
refused =
m: option:
lib.any (
a:
!a.assertion
&& lib.hasInfix "services.hyperhive.deploy.${option} is" a.message
&& lib.hasInfix "/var/lib/swarm-bao-pki/ca.pem" a.message
) m.assertions;
oldRoot = hiveWithController {
deploy.singleHostSwarm = true;
deploy.bao.clientCaFile = "/var/lib/swarm-ca/root.pem";
};
hiveCa = hive {
deploy.bao.serverCaFile = "/var/lib/hive-tls/ca.pem";
deploy.swarm-controller.hiveClientCaFile = "/var/lib/hive-tls/ca-previous.pem";
};
options = [
"bao.clientCaFile"
"bao.serverCaFile"
"swarm-controller.hiveClientCaFile"
];
in
refused oldRoot "bao.clientCaFile"
&& refused oldRoot "swarm-controller.hiveClientCaFile"
&& refused hiveCa "bao.serverCaFile"
&& refused hiveCa "swarm-controller.hiveClientCaFile"
&& !(lib.any (refused allLocal) options)
&& allLocal.services.hyperhive.deploy.bao.clientCaFile == "/var/lib/swarm-bao-pki/ca.pem";
}
{
# The store host's hive presents the leaf ./glue-bao-tls.nix mints for
# it; not the reader's, not the services issuer's.
name = "the gateway leaf is issued through this hive's own role, presenting its own identity";
ok =
let
t = allLocal.services.hyperhive.deploy.hive-controller.tls;
in
lib.hasInfix "pki/issue/swarm-hive-leaf-h1" leafUnit.script
&& lib.hasInfix "common_name=h1.t.local" leafUnit.script
&& lib.hasInfix "alt_names='*.h1.t.local'" leafUnit.script
&& lib.hasInfix "bao login -method=cert -token-only" leafUnit.script
&& leafUnit.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/hive-leaf.pem"
&& leafUnit.environment.BAO_CLIENT_KEY == "/var/lib/swarm-bao-pki/hive-leaf-key.pem"
&& t.baoHiveIdentityCertFile == leafUnit.environment.BAO_CLIENT_CERT
&& leafUnit.environment.BAO_CLIENT_CERT != t.baoClientCertFile
&& leafUnit.environment.BAO_CLIENT_CERT != allLocal.services.hyperhive.deploy.bao.clientCertFile;
}
{
# Checked before the leaf, so the unit is red even while a valid leaf
# keeps the gateway up. The control is a hive with the identity named,
# whose script checks the files instead.
name = "a hive with no store identity fails the gateway-leaf unit, naming both options";
ok =
let
s = noIdentity.systemd.services.hive-tls-resign.script;
before = lib.head (lib.splitString "reissue=0" s);
ok' = noService.systemd.services.hive-tls-resign.script;
in
lib.hasInfix "services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile (unset)" before
&& lib.hasInfix "services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityKeyFile (unset)" before
&& lib.hasInfix "exit 1" before
&& lib.hasInfix "for f in /etc/pki/bao-hive.pem /etc/pki/bao-hive-key.pem; do" (
lib.head (lib.splitString "reissue=0" ok')
)
&& !(lib.hasInfix "(unset)" ok');
}
{
# A cert from one issuance beside the key of another, left by an
# interrupted pair of renames, is re-issued rather than left for
# `nginx -t` to refuse. Checked on the decision side of the exit.
name = "the gateway leaf is re-issued when not matching its key, not chained to the store's root, or at half its role lifetime";
ok =
let
decision = lib.head (lib.splitString ''if [ "$reissue" = 0 ]'' leafUnit.script);
in
decision != leafUnit.script
&& lib.hasInfix ''openssl verify -CAfile "$svcroot" "$leaf"'' decision
&& lib.hasInfix ''leafpub="$(openssl x509 -in "$leaf" -noout -pubkey'' decision
&& lib.hasInfix ''keypub="$(openssl pkey -in "$leafk" -pubout'' decision
&& lib.hasInfix ''{ [ -n "$leafpub" ] && [ "$leafpub" = "$keypub" ]; } || reissue=1'' decision
&& lib.hasInfix (renewAt 720) leafUnit.script
&& lib.hasInfix (renewAt 48) shortTtl.systemd.services.hive-tls-resign.script
&& lib.hasInfix ''-in "$leaf" -noout -checkend "$renewat"'' leafUnit.script;
}
{
# A valid leaf on disk starts the gateway with the store sealed: nothing
# requires or waits on the issuing unit. The control is that unit
# itself, which does wait on the store's container.
name = "the gateway does not wait on the gateway-leaf unit, which waits on the store";
ok =
(leafUnit.requiredBy or [ ]) == [ ]
&& (leafUnit.before or [ ]) == [ ]
&& !(lib.any (u: lib.elem "hive-tls-resign.service" ((u.requires or [ ]) ++ (u.after or [ ]))) (
lib.attrValues (removeAttrs units [ "hive-tls-resign" ])
))
&& !(lib.elem "container@swarm-bao.service" ((caUnit.requires or [ ]) ++ (caUnit.after or [ ])))
&& lib.elem "container@swarm-bao.service" leafUnit.after
&& lib.elem "multi-user.target" leafUnit.wantedBy;
}
{
# Not `RemainAfterExit`, or the timer would start an active unit and
# renew nothing.
name = "a daily timer re-runs the gateway-leaf unit";
ok =
let
t = allLocal.systemd.timers.hive-tls-resign;
in
lib.elem "timers.target" t.wantedBy
&& t.timerConfig.OnCalendar == "daily"
&& (t.timerConfig.Unit or "hive-tls-resign.service") == "hive-tls-resign.service"
&& !(remainsAfterExit leafUnit);
}
];
in
runGroup "hive-tls" cases