vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

swarm-bao: write the first grant from the host, not the container

Browse source 
`swarm-bao-controller-policy` creates the `swarm-controller` policy and
cert-auth role — the credential every hive logs in with. It has never
succeeded on any deployment, and the reason is where it ran.

Inside the container it had neither of the two things the store demands.
Its `BAO_ADDR` was the public DNS name, which from that netns resolves to
the hive bridge: `dial tcp 10.42.0.1:8200: connect: connection refused`.
And every API listener carries `tls_require_and_verify_client_cert`, while
`tlsDir` holds the server's leaf and the CA that signs clients — no client
identity at all. Fixing only the address moves the failure one hop.

The comment above the unit asserted the opposite — that in there the store
is "reachable without a client certificate at all, which is the point". The
listener config decides that, and says otherwise. That belief is what put
the unit in the container, so it goes with it.

On the host all four coordinates already exist: `baoCli` carries the
address, the CA, the certificate and the key, so the unit needs no
`environment` block at all. `bootstrapTokenFile` was always a host path —
the container only saw it through a bind mount. Nothing new crosses the
boundary; the mount gets no wider.

The retry bound is resized with it. 10 attempts at 30s is five minutes,
and under `seal = "shamir"` an operator unseals by hand, so it would give
up before a human arrived — permanently, because `start-limit-hit` does
not self-heal. That is the same silent no-bootstrap this issue is about.
2880 × 30s covers a day, inside a 25h window.

module-eval follows the unit to the host and gains an arm asserting it is
NOT rendered inside the container: the move is the fix, so the side it
landed on is worth pinning.
This commit is contained in:
atlas 2026-09-10 21:14:23 +02:00 • committed by mara
commit 16182c670e
2 changed files with 106 additions and 95 deletions
Download patch file Download diff file Expand all files Collapse all files

41
nix/module-eval.nix
View file

@ -525,30 +525,38 @@ let
otelRemoteAuthelia.containers.swarm-otel.config.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential;
}
{
# Reads the rendered unit INSIDE the container, which is where the write
# happens: reaching the store locally is what lets the grant be written
# without a client certificate at all.
name = "a store host with a placed bootstrap token renders the granting unit inside the container";
# Reads the rendered unit on the HOST, which is where the write happens:
# every API listener demands a client certificate, and the host is the
# side that has one.
name = "a store host with a placed bootstrap token renders the granting unit on the host";
ok =
let
u = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy;
u = baoGrantHere.systemd.services.swarm-bao-controller-policy;
in
lib.hasInfix "/run/secrets/bao-bootstrap.token" u.script
&& u.unitConfig.ConditionPathExists == "/run/secrets/bao-bootstrap.token";
}
{
# The move is the fix, so pin the side it landed on: in the container it
# had no identity to open a connection with, and no address that resolved
# to the store from its own netns.
name = "the granting unit is not rendered inside the store's container";
ok = !(baoGrantHere.containers.swarm-bao.config.systemd.services ? swarm-bao-controller-policy);
}
{
# `StartLimit*` are `[Unit]` settings that systemd ignores under
# `[Service]`, so a bound written into `serviceConfig` renders, deploys
# and does nothing. Asserted where nixpkgs puts it rather than where it
# was written, and the interval is part of the bound: it has to exceed
# `RestartSec × burst` or the window closes between attempts.
# was written. The values are pinned because they are the bound: under
# `shamir` a human unseals by hand, and anything shorter than a day gives
# up first — `start-limit-hit` does not self-heal.
name = "the granting unit's start limit lands in [Unit], not [Service]";
ok =
let
u = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy;
u = baoGrantHere.systemd.services.swarm-bao-controller-policy;
in
toString u.unitConfig.StartLimitBurst == "10"
&& toString u.unitConfig.StartLimitIntervalSec == "600"
toString u.unitConfig.StartLimitBurst == "2880"
&& toString u.unitConfig.StartLimitIntervalSec == "90000"
&& !(u.serviceConfig ? StartLimitBurst);
}
{
@ -560,7 +568,7 @@ let
name = "the controller's bao grants cannot reach the policy that constrains it";
ok =
let
s = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script;
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
in
lib.hasInfix "sys/policies/acl/hive-*" s && !(lib.hasInfix "sys/policies/acl/*" s);
}
@ -574,7 +582,7 @@ let
name = "the controller may write agent credentials, and only under the agent prefix";
ok =
let
s = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script;
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
in
lib.hasInfix "secret/data/swarm/agents/*" s
&& !(lib.hasInfix "secret/data/*" s)
@ -588,7 +596,7 @@ let
name = "the controller's grant on agent credentials is write-only";
ok =
let
s = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script;
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
in
lib.hasInfix "path \"secret/data/swarm/agents/*\" {\n capabilities = [\"create\", \"update\"]" s;
}
@ -599,7 +607,7 @@ let
name = "the granting unit creates the cert auth mount and the controller's role";
ok =
let
s = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script;
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
in
lib.hasInfix "bao auth enable cert" s
&& lib.hasInfix "auth/cert/certs/swarm-controller" s
@ -752,7 +760,7 @@ let
ok =
let
m = baoControllerHere;
role = m.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script;
role = m.systemd.services.swarm-bao-controller-policy.script;
pki = m.systemd.services.swarm-bao-pki.script;
in
lib.hasInfix "cn-marker-not-a-default" role && lib.hasInfix "cn-marker-not-a-default" pki;
@ -768,8 +776,7 @@ let
name = "with no client CA the unit still writes the policy and skips the role";
ok =
let
s =
baoGrantNoClientCa.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script;
s = baoGrantNoClientCa.systemd.services.swarm-bao-controller-policy.script;
in
lib.hasInfix "bao policy write" s
&& !(lib.hasInfix "bao auth enable cert" s)