vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

hive-tls: issue the gateway leaf from bao with a per-hive identity

Browse source 
Every hive gets its own store identity (CN
<deploy.bao.hiveLeafCommonNamePrefix>-<hive>) and its own role on the
existing pki mount (<deploy.bao.hiveLeafPkiRolePrefix>-<hive>), whose
allowed_domains is the hive's domain and *.<domain>. The policy grants
update on that one issue path. swarm-bao-hive-leaf-policy writes role,
policy and cert-auth role for every hive in swarm.hives; the store's host
mints its own hive's identity in glue-bao-tls.nix and a new glue file
points deploy.hive-controller.tls.baoHiveIdentity{Cert,Key}File at it.
Hives on other hosts place it by hand. swarm.nix reserves the composed CN.

hive-tls-ca no longer generates a hive CA or signs a leaf: caGenScript,
signLeafScript/signHiveLeaf and the adoption block are gone, ca-key.pem
is deleted where it exists, and the unit only assembles trust-bundle.pem.
The retired ca.pem/ca-previous.pem stay in the bundle so leaves they
signed keep verifying.

hive-tls-resign becomes the bao issuance: at boot, on deploy and from a
daily timer it re-issues when the leaf is missing, past half the role's
lifetime, missing a name, or not chained to the store's pki root. Nothing
orders the gateway after it, so a valid leaf on disk never waits on the
store; a failed renewal leaves the leaf in place. A missing identity
fails the unit naming both options, as does the root fetch on a hive
fronting no swarm name, which now presents this identity instead of the
reader cert.

caValidityDays and leafValidityDays are removed (both spellings), since
nothing reads them.

Refs #4588
This commit is contained in:
atlas 2026-10-04 15:11:48 +02:00 • committed by mara
commit 5526daab97
16 changed files with 859 additions and 628 deletions
Download patch file Download diff file Expand all files Collapse all files

104
nix/module-eval/bao-grants.nix
View file

@ -63,6 +63,21 @@ let
deploy.bao.agentPkiRoleName = "agent";
};
# A gateway-leaf role prefix the granter's `roles/swarm-*` does not reach.
baoGranterOddHiveLeafRole = hive {
deploy.bao.enable = true;
deploy.bao.hiveLeafPkiRolePrefix = "hive-leaf";
};
# The store with a second hive in its directory, at a domain that is not
# `<name>.<swarm.domain>`, so each hive's gateway-leaf grant can be told
# apart from the other's.
baoGrantTwoHives = hive {
deploy.bao.enable = true;
deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token";
swarm.hives.h2.domain = "h2.example.org";
};
# The store and the token, with no CA to trust. `mkForce` because the PKI
# glue supplies one by default here — this is the deployment that brings its
# own certificates and has not named the authority yet, in which nothing can
@ -151,7 +166,7 @@ let
_: u: (u.environment.BAO_CLIENT_CERT or null) == granterCertFile
) baoGrantWithConsumers.systemd.services;
# The thirteen units that write a `swarm-*` grant, by name, for the discovery
# The fourteen units that write a `swarm-*` grant, by name, for the discovery
# control below.
grantingUnitNames = [
"swarm-bao-controller-policy"
@ -167,6 +182,7 @@ let
"swarm-bao-agent-pki"
"swarm-bao-nats-auth-policy"
"swarm-bao-operator-viewer-policy"
"swarm-bao-hive-leaf-policy"
];
# Comment lines dropped first: both the HCL and the scripts explain
@ -784,24 +800,24 @@ let
}
{
# A store host without the granter's pair writes its grants some other
# way, so none of the thirteen units may exist. Without this arm
# way, so none of the fourteen units may exist. Without this arm
# `lib.mkIf haveGranter` could be dropped from any of them and every other
# case here would still pass.
name = "without the granter's pair none of the thirteen granting units render";
name = "without the granter's pair none of the fourteen granting units render";
ok =
let
s = baoGranterOptOut.systemd.services;
in
lib.all (unit: !(s ? ${unit})) (grantingUnitNames ++ [ "swarm-bao-granter-role" ])
# The control: the same store with the pair renders all thirteen.
# The control: the same store with the pair renders all fourteen.
&& lib.all (unit: baoGrantHere.systemd.services ? ${unit}) grantingUnitNames;
}
{
# 🩸 What replaced the silent skip. With no bootstrap token the thirteen still
# 🩸 What replaced the silent skip. With no bootstrap token the fourteen still
# render, and a refused granter fails them with the step that fixes it.
# A store host that never named a token is told to name one, since the
# unit that sets the granter up renders only where it has.
name = "a store host without a bootstrap token renders the thirteen, each failing loudly with the one-time step";
name = "a store host without a bootstrap token renders the fourteen, each failing loudly with the one-time step";
ok =
let
s = baoGranterNoToken.systemd.services;
@ -1091,6 +1107,78 @@ let
in
lib.hasInfix "/granter.pem ]" s && lib.hasInfix "bao-granter \"\" clientAuth" s;
}
{
# One role, policy and login role per hive in the directory, each
# naming only its own hive: the policy text in the unit is exactly the
# two `update` grants, and each hive's block names neither the other's
# role nor its subject.
name = "each hive's gateway-leaf identity may issue from its own role alone, for its own domain";
ok =
let
s = baoGrantTwoHives.systemd.services.swarm-bao-hive-leaf-policy.script;
blockOf =
h: next:
lib.head (
lib.splitString "pki/roles/swarm-hive-leaf-${next}" (
lib.last (lib.splitString "pki/roles/swarm-hive-leaf-${h}" s)
)
);
h1 = blockOf "h1" "h2";
h2 = blockOf "h2" "h1";
issues =
h: dom: blk:
lib.hasInfix "allowed_domains='${dom},*.${dom}'" blk
&& lib.hasInfix "allow_ip_sans=false" blk
&& lib.hasInfix "allow_localhost=false" blk
&& lib.hasInfix "allow_any_name=false" blk
&& lib.hasInfix "max_ttl=720h" blk
&& lib.hasInfix "bao policy write swarm-hive-leaf-${h} -" blk
&& lib.hasInfix "auth/cert/certs/swarm-hive-leaf-${h}" blk
&& lib.hasInfix "allowed_common_names=swarm-bao-hive-leaf-${h}" blk
&& lib.hasInfix "token_policies=swarm-hive-leaf-${h}" blk;
in
grantsIn s == [
{
path = "pki/issue/swarm-hive-leaf-h1";
caps = [ "update" ];
}
{
path = "pki/issue/swarm-hive-leaf-h2";
caps = [ "update" ];
}
]
&& issues "h1" "h1.t.local" h1
&& issues "h2" "h2.example.org" h2
&& !(lib.hasInfix "h2" h1)
&& !(lib.hasInfix "h1" h2);
}
{
# The minting side, and the hive's own path to it: the store host signs
# its hive's leaf under the composed subject and points the gateway-leaf
# unit at it.
name = "the PKI unit signs this hive's gateway-leaf identity, and the hive presents it";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-pki.script;
t = baoGrantHere.services.hyperhive.deploy.hive-controller.tls;
in
lib.hasInfix "/hive-leaf.pem ]" s
&& lib.hasInfix "swarm-bao-hive-leaf-h1 \"\" clientAuth" s
&& t.baoHiveIdentityCertFile == "/var/lib/swarm-bao-pki/hive-leaf.pem"
&& t.baoHiveIdentityKeyFile == "/var/lib/swarm-bao-pki/hive-leaf-key.pem"
# Its own leaf, not the reader's or the services issuer's.
&& t.baoHiveIdentityCertFile != baoGrantHere.services.hyperhive.deploy.bao.clientCertFile
&& t.baoHiveIdentityCertFile != t.baoClientCertFile;
}
{
name = "a gateway-leaf role prefix outside swarm-* is refused, naming the option";
ok =
let
names = a: lib.hasInfix "services.hyperhive.deploy.bao.hiveLeafPkiRolePrefix" a.message;
in
lib.any (a: !a.assertion && names a) baoGranterOddHiveLeafRole.assertions
&& !(lib.any (a: !a.assertion && names a) baoGrantHere.assertions);
}
{
# The granter writes pki roles through `roles/swarm-*` only, so a role
# named otherwise is refused at eval rather than 403'd at deploy.
@ -1304,8 +1392,8 @@ let
}
{
# What makes the case above mean something: discovery by the granter's
# certificate reaches all thirteen units, and each yields calls.
name = "the granter-policy check sees all thirteen granting units, and parses calls from each";
# certificate reaches all fourteen units, and each yields calls.
name = "the granter-policy check sees all fourteen granting units, and parses calls from each";
ok =
lib.sort lib.lessThan (lib.attrNames granterUnits) == lib.sort lib.lessThan grantingUnitNames
&& lib.all (u: baoCalls u.script != [ ]) (lib.attrValues granterUnits)

145
nix/module-eval/hive-tls.nix
View file

@ -3,7 +3,8 @@
#
# The swarm-services leaf's renewal: a timer that really re-runs the
# issuance, at a threshold read off the store role's lifetime. And the
# store's root on a hive that asks it for no leaf.
# store's root on a hive that asks it for no leaf. And the gateway leaf,
# issued by the store through the hive's own role, with no hive CA key.
{
pkgs,
lib,
@ -38,15 +39,17 @@ let
};
# Fronting none of the swarm's names, reading a store it does not run,
# with the reader identity placed by hand.
# with the reader identity and the hive's own identity placed by hand.
noService = hive {
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
deploy.bao.serverCaFile = "/etc/pki/bao-ca.pem";
deploy.hive-controller.tls.baoHiveIdentityCertFile = "/etc/pki/bao-hive.pem";
deploy.hive-controller.tls.baoHiveIdentityKeyFile = "/etc/pki/bao-hive-key.pem";
};
noSvcFetch = noService.systemd.services.swarm-services-cert.script;
# The same hive with no reader identity at all.
# The same hive with no store identity at all.
noIdentity = hive { };
noIdFetch = noIdentity.systemd.services.swarm-services-cert.script;
@ -60,6 +63,8 @@ let
units = allLocal.systemd.services;
boot = units.swarm-services-cert;
caUnit = units.hive-tls-ca;
leafUnit = units.hive-tls-resign;
timer = allLocal.systemd.timers.swarm-services-cert-renew;
# What the timer starts, read off the timer rather than assumed from its
@ -151,13 +156,16 @@ let
&& lib.hasInfix ''if [ -s "$svcroot" ]; then anchors+=("$svcroot"); fi'' noService.systemd.services.hive-tls-ca.script;
}
{
name = "the root fetch reaches the store with the hive's own reader identity and server CA";
# The hive's own identity, not its reader identity: the reader's policy
# reads every secret the hive is entitled to.
name = "the root fetch reaches the store with the hive's own identity and server CA";
ok =
lib.hasInfix "--cacert ${lib.escapeShellArg "/etc/pki/bao-ca.pem"}" noSvcFetch
&& lib.hasInfix (
"--cert ${lib.escapeShellArg "/etc/pki/bao-client.pem"}"
+ " --key ${lib.escapeShellArg "/etc/pki/bao-client-key.pem"}"
) noSvcFetch;
"--cert ${lib.escapeShellArg "/etc/pki/bao-hive.pem"}"
+ " --key ${lib.escapeShellArg "/etc/pki/bao-hive-key.pem"}"
) noSvcFetch
&& !(lib.hasInfix "/etc/pki/bao-client.pem" (noSvcBranch noSvcFetch));
}
{
# No identity to present: the unit fails, naming both options, rather
@ -166,11 +174,11 @@ let
name = "a hive fronting no swarm name with no store identity fails the unit, naming both options";
ok =
noIdentity.services.hyperhive.swarm.localServiceDomains == [ ]
&& noIdentity.services.hyperhive.deploy.bao.clientCertFile == null
&& noIdentity.services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile == null
&& !(lib.hasInfix "/v1/pki/ca/pem" noIdFetch)
&& lib.hasInfix "services.hyperhive.deploy.bao.clientCertFile" (noSvcBranch noIdFetch)
&& lib.hasInfix "services.hyperhive.deploy.bao.clientKeyFile" (noSvcBranch noIdFetch)
&& lib.hasSuffix "exit 1" (noSvcBranch noIdFetch)
&& lib.hasInfix "baoHiveIdentityCertFile (unset)" (noSvcBranch noIdFetch)
&& lib.hasInfix "baoHiveIdentityKeyFile (unset)" (noSvcBranch noIdFetch)
&& lib.hasSuffix "exit 1" (lib.removeSuffix "\n" (noSvcBranch noIdFetch))
&& !(lib.hasInfix "exit 0" (noSvcBranch noIdFetch))
&& !(lib.hasInfix "not staged" noIdFetch)
&& lib.hasInfix "/v1/pki/ca/pem" (noSvcBranch noSvcFetch);
@ -179,13 +187,11 @@ let
# An identity named but not on disk fails before curl, naming the paths.
name = "a hive fronting no swarm name whose identity files are absent fails the unit, naming the paths";
ok =
lib.hasInfix "for f in /etc/pki/bao-client.pem /etc/pki/bao-client-key.pem; do" (
noSvcBranch noSvcFetch
)
lib.hasInfix "for f in /etc/pki/bao-hive.pem /etc/pki/bao-hive-key.pem; do" (noSvcBranch noSvcFetch)
&& lib.hasInfix ''[ -r "$f" ] || missing="$missing $f"'' (noSvcBranch noSvcFetch)
&& lib.hasInfix "cannot read$missing" (noSvcBranch noSvcFetch)
&& lib.hasInfix "clientCertFile (/etc/pki/bao-client.pem)" (noSvcBranch noSvcFetch)
&& lib.hasInfix "clientKeyFile (/etc/pki/bao-client-key.pem)" (noSvcBranch noSvcFetch);
&& lib.hasInfix "baoHiveIdentityCertFile (/etc/pki/bao-hive.pem)" (noSvcBranch noSvcFetch)
&& lib.hasInfix "baoHiveIdentityKeyFile (/etc/pki/bao-hive-key.pem)" (noSvcBranch noSvcFetch);
}
{
# The fetch retries against a sealed store; a gateway `Requires=` on it
@ -201,6 +207,113 @@ let
in
!(holds noService) && holds allLocal;
}
{
# The controls are the store's PKI unit, which still mints its own CA
# with `openssl req`, and the swarm root's unit, which still reads
# `root-key.pem`: absence here is not a matcher that cannot match.
name = "no hive TLS unit generates a CA, signs with a CA key or reads the swarm root key";
ok =
let
hasCaWork =
s:
lib.any (p: lib.hasInfix p s) [
"openssl req"
"-CAkey"
"root-key.pem"
];
pkiScript = units.swarm-bao-pki.script;
rootScript = units.swarm-ca.script;
in
!(hasCaWork caUnit.script)
&& !(hasCaWork leafUnit.script)
&& !(hasCaWork noService.systemd.services.hive-tls-ca.script)
&& lib.hasInfix "openssl req" pkiScript
&& lib.hasInfix "root-key.pem" rootScript;
}
{
name = "a retired hive CA key is deleted, and its public certificate stays in the trust bundle";
ok =
lib.hasInfix ''rm -f "$d/ca-key.pem"'' caUnit.script
&& lib.hasInfix ''ca="$d/ca.pem"'' caUnit.script
&& lib.hasInfix ''if [ -s "$ca" ]; then anchors+=("$ca"); fi'' caUnit.script
&& lib.hasInfix ''if [ -s "$prev" ]; then anchors+=("$prev"); fi'' caUnit.script
&& lib.hasInfix ''cat "''${anchors[@]}" > "$bundle"'' caUnit.script;
}
{
# The store host's hive presents the leaf ./glue-bao-tls.nix mints for
# it; not the reader's, not the services issuer's.
name = "the gateway leaf is issued through this hive's own role, presenting its own identity";
ok =
let
t = allLocal.services.hyperhive.deploy.hive-controller.tls;
in
lib.hasInfix "pki/issue/swarm-hive-leaf-h1" leafUnit.script
&& lib.hasInfix "common_name=h1.t.local" leafUnit.script
&& lib.hasInfix "alt_names='*.h1.t.local'" leafUnit.script
&& lib.hasInfix "bao login -method=cert -token-only" leafUnit.script
&& leafUnit.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/hive-leaf.pem"
&& leafUnit.environment.BAO_CLIENT_KEY == "/var/lib/swarm-bao-pki/hive-leaf-key.pem"
&& t.baoHiveIdentityCertFile == leafUnit.environment.BAO_CLIENT_CERT
&& leafUnit.environment.BAO_CLIENT_CERT != t.baoClientCertFile
&& leafUnit.environment.BAO_CLIENT_CERT != allLocal.services.hyperhive.deploy.bao.clientCertFile;
}
{
# Checked before the leaf, so the unit is red even while a valid leaf
# keeps the gateway up. The control is a hive with the identity named,
# whose script checks the files instead.
name = "a hive with no store identity fails the gateway-leaf unit, naming both options";
ok =
let
s = noIdentity.systemd.services.hive-tls-resign.script;
before = lib.head (lib.splitString "reissue=0" s);
ok' = noService.systemd.services.hive-tls-resign.script;
in
lib.hasInfix "services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile (unset)" before
&& lib.hasInfix "services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityKeyFile (unset)" before
&& lib.hasInfix "exit 1" before
&& lib.hasInfix "for f in /etc/pki/bao-hive.pem /etc/pki/bao-hive-key.pem; do" (
lib.head (lib.splitString "reissue=0" ok')
)
&& !(lib.hasInfix "(unset)" ok');
}
{
# A leaf the retired hive CA signed is re-issued from the store on the
# first run that reaches it; one that renews in time never is.
name = "the gateway leaf is re-issued when not chained to the store's root, and at half its role lifetime";
ok =
lib.hasInfix ''openssl verify -CAfile "$svcroot" "$leaf"'' leafUnit.script
&& lib.hasInfix (renewAt 720) leafUnit.script
&& lib.hasInfix (renewAt 48) shortTtl.systemd.services.hive-tls-resign.script
&& lib.hasInfix ''-in "$leaf" -noout -checkend "$renewat"'' leafUnit.script;
}
{
# A valid leaf on disk starts the gateway with the store sealed: nothing
# requires or waits on the issuing unit. The control is that unit
# itself, which does wait on the store's container.
name = "the gateway does not wait on the gateway-leaf unit, which waits on the store";
ok =
(leafUnit.requiredBy or [ ]) == [ ]
&& (leafUnit.before or [ ]) == [ ]
&& !(lib.any (u: lib.elem "hive-tls-resign.service" ((u.requires or [ ]) ++ (u.after or [ ]))) (
lib.attrValues (removeAttrs units [ "hive-tls-resign" ])
))
&& !(lib.elem "container@swarm-bao.service" ((caUnit.requires or [ ]) ++ (caUnit.after or [ ])))
&& lib.elem "container@swarm-bao.service" leafUnit.after
&& lib.elem "multi-user.target" leafUnit.wantedBy;
}
{
# Not `RemainAfterExit`, or the timer would start an active unit and
# renew nothing.
name = "a daily timer re-runs the gateway-leaf unit";
ok =
let
t = allLocal.systemd.timers.hive-tls-resign;
in
lib.elem "timers.target" t.wantedBy
&& t.timerConfig.OnCalendar == "daily"
&& (t.timerConfig.Unit or "hive-tls-resign.service") == "hive-tls-resign.service"
&& !(remainsAfterExit leafUnit);
}
];
in
runGroup "hive-tls" cases

19
nix/module-eval/name-guards.nix
View file

@ -106,6 +106,14 @@ let
swarm.hives.qr-other.domain = "q.t.local";
};
# The per-hive gateway-leaf subject, composed the same way.
hiveNamedAfterHiveLeafSubject = hive {
deploy.swarm-otel.enable = false;
deploy.bao.hiveLeafCommonNamePrefix = "hl";
swarm.hives.other.domain = "o.t.local";
swarm.hives.hl-other.domain = "hl.t.local";
};
hiveNameWithComposedWord = hive {
deploy.swarm-otel.enable = false;
swarm.hives."h1-agent".domain = "a.t.local";
@ -203,6 +211,17 @@ let
a: !a.assertion && lib.hasInfix "'qr-other'" a.message
) hiveNamedAfterPerHiveReaderSubject.assertions;
}
{
# A hive taking this spelling would present a leaf hive `other`'s
# gateway-leaf role accepts, and could ask the store for that hive's
# certificate. The control is the legal roster below.
name = "a hive named after another hive's gateway-leaf subject is refused";
ok =
equalityGuardFired hiveNamedAfterHiveLeafSubject
&& lib.any (
a: !a.assertion && lib.hasInfix "'hl-other'" a.message
) hiveNamedAfterHiveLeafSubject.assertions;
}
{
# Without this the case above proves nothing: an arm that fires for every
# roster is not a guard, and `hives` is non-empty in both fixtures.