5526daab97
Every hive gets its own store identity (CN
<deploy.bao.hiveLeafCommonNamePrefix>-<hive>) and its own role on the
existing pki mount (<deploy.bao.hiveLeafPkiRolePrefix>-<hive>), whose
allowed_domains is the hive's domain and *.<domain>. The policy grants
update on that one issue path. swarm-bao-hive-leaf-policy writes role,
policy and cert-auth role for every hive in swarm.hives; the store's host
mints its own hive's identity in glue-bao-tls.nix and a new glue file
points deploy.hive-controller.tls.baoHiveIdentity{Cert,Key}File at it.
Hives on other hosts place it by hand. swarm.nix reserves the composed CN.
hive-tls-ca no longer generates a hive CA or signs a leaf: caGenScript,
signLeafScript/signHiveLeaf and the adoption block are gone, ca-key.pem
is deleted where it exists, and the unit only assembles trust-bundle.pem.
The retired ca.pem/ca-previous.pem stay in the bundle so leaves they
signed keep verifying.
hive-tls-resign becomes the bao issuance: at boot, on deploy and from a
daily timer it re-issues when the leaf is missing, past half the role's
lifetime, missing a name, or not chained to the store's pki root. Nothing
orders the gateway after it, so a valid leaf on disk never waits on the
store; a failed renewal leaves the leaf in place. A missing identity
fails the unit naming both options, as does the root fetch on a hive
fronting no swarm name, which now presents this identity instead of the
reader cert.
caValidityDays and leafValidityDays are removed (both spellings), since
nothing reads them.
Refs #4588
319 lines
15 KiB
Nix
319 lines
15 KiB
Nix
# `checks.module-eval-hive-tls` — see ./lib.nix for the shared rationale (why
|
|
# this suite exists, naming convention, "evaluates not executes").
|
|
#
|
|
# The swarm-services leaf's renewal: a timer that really re-runs the
|
|
# issuance, at a threshold read off the store role's lifetime. And the
|
|
# store's root on a hive that asks it for no leaf. And the gateway leaf,
|
|
# issued by the store through the hive's own role, with no hive CA key.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
runGroup
|
|
;
|
|
|
|
# Every service on one host, so the store's granting unit renders the role
|
|
# this leaf is issued through.
|
|
allLocal = hive {
|
|
deploy.singleHostSwarm = true;
|
|
};
|
|
|
|
# The same host with the role's lifetime moved, so a threshold that is a
|
|
# number of its own shows up as one that did not move with it.
|
|
shortTtl = hive {
|
|
deploy.singleHostSwarm = true;
|
|
deploy.bao.servicesPkiLeafTtlHours = 48;
|
|
};
|
|
|
|
# Fronting none of the swarm's names, reading a store it does not run,
|
|
# with the reader identity and the hive's own identity placed by hand.
|
|
noService = hive {
|
|
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
|
|
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
|
|
deploy.bao.serverCaFile = "/etc/pki/bao-ca.pem";
|
|
deploy.hive-controller.tls.baoHiveIdentityCertFile = "/etc/pki/bao-hive.pem";
|
|
deploy.hive-controller.tls.baoHiveIdentityKeyFile = "/etc/pki/bao-hive-key.pem";
|
|
};
|
|
noSvcFetch = noService.systemd.services.swarm-services-cert.script;
|
|
|
|
# The same hive with no store identity at all.
|
|
noIdentity = hive { };
|
|
noIdFetch = noIdentity.systemd.services.swarm-services-cert.script;
|
|
|
|
# The `want_svc=0` branch of a rendered script, up to the `fi` that closes it (the
|
|
# only one followed by a blank line).
|
|
noSvcBranch =
|
|
script:
|
|
builtins.head (
|
|
lib.splitString "\nfi\n\n" (lib.last (lib.splitString ''if [ "$want_svc" = 0 ]; then'' script))
|
|
);
|
|
|
|
units = allLocal.systemd.services;
|
|
boot = units.swarm-services-cert;
|
|
caUnit = units.hive-tls-ca;
|
|
leafUnit = units.hive-tls-resign;
|
|
timer = allLocal.systemd.timers.swarm-services-cert-renew;
|
|
|
|
# What the timer starts, read off the timer rather than assumed from its
|
|
# name: a timer's `Unit=` defaults to its own name, and pointing it at the
|
|
# boot unit is exactly the regression the cases below exist for.
|
|
triggeredName = lib.removeSuffix ".service" (
|
|
timer.timerConfig.Unit or "swarm-services-cert-renew.service"
|
|
);
|
|
triggered = units.${triggeredName};
|
|
|
|
remainsAfterExit = u: u.serviceConfig.RemainAfterExit or false;
|
|
|
|
renewAt = hours: "renewat=$(( ${toString hours} * 3600 / 2 ))";
|
|
|
|
cases = [
|
|
{
|
|
# Control: the boot issuance is the unit a timer cannot re-run. Were it
|
|
# not `RemainAfterExit`, the case after next would pass vacuously.
|
|
name = "the boot issuance renders, and stays active after it exits";
|
|
ok = lib.hasInfix "pki/issue/swarm-services" boot.script && remainsAfterExit boot;
|
|
}
|
|
{
|
|
name = "a timer for the services leaf is enabled and fires daily";
|
|
ok =
|
|
lib.elem "timers.target" timer.wantedBy
|
|
&& timer.timerConfig.OnCalendar == "daily"
|
|
&& timer.timerConfig.Persistent;
|
|
}
|
|
{
|
|
# Starting an active unit is a no-op, so a timer aimed at a
|
|
# `RemainAfterExit` unit fires on schedule and renews nothing.
|
|
name = "the timer starts a unit that re-runs the issuance and does not stay active";
|
|
ok =
|
|
units ? ${triggeredName}
|
|
&& !(remainsAfterExit triggered)
|
|
&& triggered.script == boot.script
|
|
# The whole set, `PATH` included: an environment copied off the
|
|
# boot unit's merged options renders a second `PATH` and fails.
|
|
&& triggered.environment == boot.environment;
|
|
}
|
|
{
|
|
# A restart of anything the gateway's cert import `Requires=` takes
|
|
# nginx down with it, so the renewal must be something nothing else
|
|
# depends on, and must run behind the boot issuance.
|
|
name = "nothing requires or waits on the renewal, and it runs after the boot issuance";
|
|
ok =
|
|
(triggered.requiredBy or [ ]) == [ ]
|
|
&& (triggered.wantedBy or [ ]) == [ ]
|
|
&& (triggered.before or [ ]) == [ ]
|
|
&& lib.elem "swarm-services-cert.service" triggered.after
|
|
&& !(lib.any (u: lib.elem "${triggeredName}.service" ((u.requires or [ ]) ++ (u.after or [ ]))) (
|
|
lib.attrValues (removeAttrs units [ triggeredName ])
|
|
));
|
|
}
|
|
{
|
|
# Moved with the role, in both places: the threshold is half of what the
|
|
# store grants, and the store grants what the option says.
|
|
name = "the leaf is renewed at half the role's lifetime, read from the option";
|
|
ok =
|
|
lib.hasInfix (renewAt 720) boot.script
|
|
&& lib.hasInfix (renewAt 48) shortTtl.systemd.services.swarm-services-cert.script
|
|
&& lib.hasInfix ''-in "$svcleaf" -noout -checkend "$renewat"'' boot.script
|
|
&& lib.hasInfix "max_ttl=48h" shortTtl.systemd.services.swarm-bao-controller-policy.script;
|
|
}
|
|
{
|
|
# The store replaces its root once it has less than one leaf lifetime
|
|
# left, and the hive asks for a fresh leaf, with the new root, at that
|
|
# same threshold. A number of its own here keeps a leaf whose root the
|
|
# store has already replaced.
|
|
name = "the services root is re-checked at the store's own replacement threshold";
|
|
ok =
|
|
let
|
|
shortBoot = shortTtl.systemd.services.swarm-services-cert.script;
|
|
shortStore = shortTtl.systemd.services.swarm-bao-controller-policy.script;
|
|
in
|
|
lib.hasInfix ''-in "$svcroot" -noout -checkend ${toString (720 * 3600)} '' boot.script
|
|
&& lib.hasInfix ''-in "$svcroot" -noout -checkend ${toString (48 * 3600)} '' shortBoot
|
|
&& lib.hasInfix "-checkend ${toString (48 * 3600)} <<<\"$root_ca\"" shortStore;
|
|
}
|
|
{
|
|
# The slot the bundle reads the store's root from, filled on a hive
|
|
# that asks the store for no leaf.
|
|
name = "a hive fronting no swarm name puts the store's pki root in its trust bundle";
|
|
ok =
|
|
noService.services.hyperhive.swarm.localServiceDomains == [ ]
|
|
&& lib.hasInfix ''"$BAO_ADDR/v1/pki/ca/pem"'' noSvcFetch
|
|
&& lib.hasInfix ''mv -f "$svcroot.new" "$svcroot"'' noSvcFetch
|
|
&& lib.hasInfix ''svcroot="$d/swarm-services-root.pem"'' noService.systemd.services.hive-tls-ca.script
|
|
&& lib.hasInfix ''if [ -s "$svcroot" ]; then anchors+=("$svcroot"); fi'' noService.systemd.services.hive-tls-ca.script;
|
|
}
|
|
{
|
|
# The hive's own identity, not its reader identity: the reader's policy
|
|
# reads every secret the hive is entitled to.
|
|
name = "the root fetch reaches the store with the hive's own identity and server CA";
|
|
ok =
|
|
lib.hasInfix "--cacert ${lib.escapeShellArg "/etc/pki/bao-ca.pem"}" noSvcFetch
|
|
&& lib.hasInfix (
|
|
"--cert ${lib.escapeShellArg "/etc/pki/bao-hive.pem"}"
|
|
+ " --key ${lib.escapeShellArg "/etc/pki/bao-hive-key.pem"}"
|
|
) noSvcFetch
|
|
&& !(lib.hasInfix "/etc/pki/bao-client.pem" (noSvcBranch noSvcFetch));
|
|
}
|
|
{
|
|
# No identity to present: the unit fails, naming both options, rather
|
|
# than leaving the bundle without the root behind a green unit. The
|
|
# control is the same hive with an identity, whose branch fetches.
|
|
name = "a hive fronting no swarm name with no store identity fails the unit, naming both options";
|
|
ok =
|
|
noIdentity.services.hyperhive.swarm.localServiceDomains == [ ]
|
|
&& noIdentity.services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile == null
|
|
&& !(lib.hasInfix "/v1/pki/ca/pem" noIdFetch)
|
|
&& lib.hasInfix "baoHiveIdentityCertFile (unset)" (noSvcBranch noIdFetch)
|
|
&& lib.hasInfix "baoHiveIdentityKeyFile (unset)" (noSvcBranch noIdFetch)
|
|
&& lib.hasSuffix "exit 1" (lib.removeSuffix "\n" (noSvcBranch noIdFetch))
|
|
&& !(lib.hasInfix "exit 0" (noSvcBranch noIdFetch))
|
|
&& !(lib.hasInfix "not staged" noIdFetch)
|
|
&& lib.hasInfix "/v1/pki/ca/pem" (noSvcBranch noSvcFetch);
|
|
}
|
|
{
|
|
# An identity named but not on disk fails before curl, naming the paths.
|
|
name = "a hive fronting no swarm name whose identity files are absent fails the unit, naming the paths";
|
|
ok =
|
|
lib.hasInfix "for f in /etc/pki/bao-hive.pem /etc/pki/bao-hive-key.pem; do" (noSvcBranch noSvcFetch)
|
|
&& lib.hasInfix ''[ -r "$f" ] || missing="$missing $f"'' (noSvcBranch noSvcFetch)
|
|
&& lib.hasInfix "cannot read$missing" (noSvcBranch noSvcFetch)
|
|
&& lib.hasInfix "baoHiveIdentityCertFile (/etc/pki/bao-hive.pem)" (noSvcBranch noSvcFetch)
|
|
&& lib.hasInfix "baoHiveIdentityKeyFile (/etc/pki/bao-hive-key.pem)" (noSvcBranch noSvcFetch);
|
|
}
|
|
{
|
|
# The fetch retries against a sealed store; a gateway `Requires=` on it
|
|
# would go down with each attempt. The control is a host fronting
|
|
# swarm names, whose gateway does wait for its leaf.
|
|
name = "a hive fronting no swarm name does not hold its gateway on the store";
|
|
ok =
|
|
let
|
|
holds =
|
|
m:
|
|
lib.elem "hive-gateway-self-signed-cert.service" m.systemd.services.swarm-services-cert.requiredBy
|
|
|| lib.elem "hive-gateway-self-signed-cert.service" m.systemd.services.swarm-services-cert.before;
|
|
in
|
|
!(holds noService) && holds allLocal;
|
|
}
|
|
{
|
|
# The controls are the store's PKI unit, which still mints its own CA
|
|
# with `openssl req`, and the swarm root's unit, which still reads
|
|
# `root-key.pem`: absence here is not a matcher that cannot match.
|
|
name = "no hive TLS unit generates a CA, signs with a CA key or reads the swarm root key";
|
|
ok =
|
|
let
|
|
hasCaWork =
|
|
s:
|
|
lib.any (p: lib.hasInfix p s) [
|
|
"openssl req"
|
|
"-CAkey"
|
|
"root-key.pem"
|
|
];
|
|
pkiScript = units.swarm-bao-pki.script;
|
|
rootScript = units.swarm-ca.script;
|
|
in
|
|
!(hasCaWork caUnit.script)
|
|
&& !(hasCaWork leafUnit.script)
|
|
&& !(hasCaWork noService.systemd.services.hive-tls-ca.script)
|
|
&& lib.hasInfix "openssl req" pkiScript
|
|
&& lib.hasInfix "root-key.pem" rootScript;
|
|
}
|
|
{
|
|
name = "a retired hive CA key is deleted, and its public certificate stays in the trust bundle";
|
|
ok =
|
|
lib.hasInfix ''rm -f "$d/ca-key.pem"'' caUnit.script
|
|
&& lib.hasInfix ''ca="$d/ca.pem"'' caUnit.script
|
|
&& lib.hasInfix ''if [ -s "$ca" ]; then anchors+=("$ca"); fi'' caUnit.script
|
|
&& lib.hasInfix ''if [ -s "$prev" ]; then anchors+=("$prev"); fi'' caUnit.script
|
|
&& lib.hasInfix ''cat "''${anchors[@]}" > "$bundle"'' caUnit.script;
|
|
}
|
|
{
|
|
# The store host's hive presents the leaf ./glue-bao-tls.nix mints for
|
|
# it; not the reader's, not the services issuer's.
|
|
name = "the gateway leaf is issued through this hive's own role, presenting its own identity";
|
|
ok =
|
|
let
|
|
t = allLocal.services.hyperhive.deploy.hive-controller.tls;
|
|
in
|
|
lib.hasInfix "pki/issue/swarm-hive-leaf-h1" leafUnit.script
|
|
&& lib.hasInfix "common_name=h1.t.local" leafUnit.script
|
|
&& lib.hasInfix "alt_names='*.h1.t.local'" leafUnit.script
|
|
&& lib.hasInfix "bao login -method=cert -token-only" leafUnit.script
|
|
&& leafUnit.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/hive-leaf.pem"
|
|
&& leafUnit.environment.BAO_CLIENT_KEY == "/var/lib/swarm-bao-pki/hive-leaf-key.pem"
|
|
&& t.baoHiveIdentityCertFile == leafUnit.environment.BAO_CLIENT_CERT
|
|
&& leafUnit.environment.BAO_CLIENT_CERT != t.baoClientCertFile
|
|
&& leafUnit.environment.BAO_CLIENT_CERT != allLocal.services.hyperhive.deploy.bao.clientCertFile;
|
|
}
|
|
{
|
|
# Checked before the leaf, so the unit is red even while a valid leaf
|
|
# keeps the gateway up. The control is a hive with the identity named,
|
|
# whose script checks the files instead.
|
|
name = "a hive with no store identity fails the gateway-leaf unit, naming both options";
|
|
ok =
|
|
let
|
|
s = noIdentity.systemd.services.hive-tls-resign.script;
|
|
before = lib.head (lib.splitString "reissue=0" s);
|
|
ok' = noService.systemd.services.hive-tls-resign.script;
|
|
in
|
|
lib.hasInfix "services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile (unset)" before
|
|
&& lib.hasInfix "services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityKeyFile (unset)" before
|
|
&& lib.hasInfix "exit 1" before
|
|
&& lib.hasInfix "for f in /etc/pki/bao-hive.pem /etc/pki/bao-hive-key.pem; do" (
|
|
lib.head (lib.splitString "reissue=0" ok')
|
|
)
|
|
&& !(lib.hasInfix "(unset)" ok');
|
|
}
|
|
{
|
|
# A leaf the retired hive CA signed is re-issued from the store on the
|
|
# first run that reaches it; one that renews in time never is.
|
|
name = "the gateway leaf is re-issued when not chained to the store's root, and at half its role lifetime";
|
|
ok =
|
|
lib.hasInfix ''openssl verify -CAfile "$svcroot" "$leaf"'' leafUnit.script
|
|
&& lib.hasInfix (renewAt 720) leafUnit.script
|
|
&& lib.hasInfix (renewAt 48) shortTtl.systemd.services.hive-tls-resign.script
|
|
&& lib.hasInfix ''-in "$leaf" -noout -checkend "$renewat"'' leafUnit.script;
|
|
}
|
|
{
|
|
# A valid leaf on disk starts the gateway with the store sealed: nothing
|
|
# requires or waits on the issuing unit. The control is that unit
|
|
# itself, which does wait on the store's container.
|
|
name = "the gateway does not wait on the gateway-leaf unit, which waits on the store";
|
|
ok =
|
|
(leafUnit.requiredBy or [ ]) == [ ]
|
|
&& (leafUnit.before or [ ]) == [ ]
|
|
&& !(lib.any (u: lib.elem "hive-tls-resign.service" ((u.requires or [ ]) ++ (u.after or [ ]))) (
|
|
lib.attrValues (removeAttrs units [ "hive-tls-resign" ])
|
|
))
|
|
&& !(lib.elem "container@swarm-bao.service" ((caUnit.requires or [ ]) ++ (caUnit.after or [ ])))
|
|
&& lib.elem "container@swarm-bao.service" leafUnit.after
|
|
&& lib.elem "multi-user.target" leafUnit.wantedBy;
|
|
}
|
|
{
|
|
# Not `RemainAfterExit`, or the timer would start an active unit and
|
|
# renew nothing.
|
|
name = "a daily timer re-runs the gateway-leaf unit";
|
|
ok =
|
|
let
|
|
t = allLocal.systemd.timers.hive-tls-resign;
|
|
in
|
|
lib.elem "timers.target" t.wantedBy
|
|
&& t.timerConfig.OnCalendar == "daily"
|
|
&& (t.timerConfig.Unit or "hive-tls-resign.service") == "hive-tls-resign.service"
|
|
&& !(remainsAfterExit leafUnit);
|
|
}
|
|
];
|
|
in
|
|
runGroup "hive-tls" cases
|