Compare commits
3 changed files with 94 additions and 127 deletions
|
|
@ -103,17 +103,16 @@ bao token create -policy=swarm-bootstrap -ttl=24h -orphan -display-name=swarm-bo
|
|||
```
|
||||
|
||||
Put the token's value at `services.hyperhive.deploy.bao.bootstrapTokenFile`
|
||||
(all-local names that path for you), then rebuild. A one-shot unit **on the
|
||||
host** reads it, writes the `swarm-controller` policy, enables the cert auth
|
||||
method, and creates the `swarm-controller` role that attaches the two. It runs
|
||||
there because every API listener demands a client certificate, and the host is
|
||||
the side that has one.
|
||||
(all-local names that path for you), then rebuild. A one-shot unit inside the
|
||||
store's container reads it, writes the `swarm-controller` policy, enables the
|
||||
cert auth method, and creates the `swarm-controller` role that attaches the two.
|
||||
|
||||
⚠️ **The grants above are derived, not proven.** They come from
|
||||
`-output-policy`, not from a swarm that came up on them. The unit has run twice
|
||||
against a live store and failed both times for reasons of its own — it could
|
||||
not reach the store from where it then ran — so neither run exercised these
|
||||
grants at all.
|
||||
⚠️ **This has only run once, and it didn't go all the way through.** The
|
||||
first real provision against a live store returned 403, so
|
||||
treat the block above as derived, then only partly tested — not proven: the
|
||||
grants still come from `-output-policy`, not from a swarm that came up on them.
|
||||
What that 403 does _not_ tell you is whether the login or the write was
|
||||
refused — `bao read auth/cert/certs/swarm-controller` separates the two.
|
||||
|
||||
**Delete the file only once `bao read auth/cert/certs/swarm-controller` returns
|
||||
the role.** The unit skips when the token is absent, so a host that has finished
|
||||
|
|
|
|||
|
|
@ -856,88 +856,6 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
# The swarm's first grant, written from the HOST. Every API listener sets
|
||||
# `tls_require_and_verify_client_cert`, so a client needs an identity
|
||||
# wherever it runs — and only the host has one. The bootstrap token is a
|
||||
# host path too; the container saw it through a bind mount.
|
||||
systemd.services.swarm-bao-controller-policy = lib.mkIf haveBootstrapToken {
|
||||
description = "write the swarm controller's bao policy and cert-auth role";
|
||||
after = [ "container@${cfg.machine}.service" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
# The wrapper rather than the package: it carries the address, the CA
|
||||
# and this host's certificate, which is what makes running here cheaper
|
||||
# than shipping an identity the other way.
|
||||
path = [
|
||||
baoCli
|
||||
pkgs.coreutils
|
||||
];
|
||||
# Named but not placed is a legitimate state: all-local supplies the
|
||||
# path as a default and the operator drops the file there after
|
||||
# `bao operator init`. Skipping rather than failing is also what makes
|
||||
# deleting the token at the end of that procedure safe.
|
||||
unitConfig.ConditionPathExists = baoDeploy.bootstrapTokenFile;
|
||||
# A store that is up is not necessarily unsealed — under
|
||||
# `seal = "shamir"` an operator unseals BY HAND, so early attempts fail
|
||||
# for as long as that takes, which can be a day.
|
||||
#
|
||||
# `StartLimit*` are `[Unit]` settings that systemd ignores under
|
||||
# `[Service]`, which is what these two options render to. The window has
|
||||
# to exceed `RestartSec × burst`, or it closes between attempts and the
|
||||
# burst is never reached: 2880 × 30s is 24h, inside a 25h window.
|
||||
# Bounding it tighter would give up before a human could unseal, and
|
||||
# `start-limit-hit` does not self-heal.
|
||||
startLimitBurst = 2880;
|
||||
startLimitIntervalSec = 90000;
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
Restart = "on-failure";
|
||||
RestartSec = 30;
|
||||
};
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
BAO_TOKEN="$(cat ${lib.escapeShellArg baoDeploy.bootstrapTokenFile})"
|
||||
export BAO_TOKEN
|
||||
|
||||
# Idempotent on purpose: a rebuild re-asserts the policy rather
|
||||
# than failing on one that already exists.
|
||||
printf '%s' ${lib.escapeShellArg controllerPolicyText} |
|
||||
bao policy write ${lib.escapeShellArg controllerPolicyName} -
|
||||
''
|
||||
+ lib.optionalString (baoDeploy.clientCaFile != null) ''
|
||||
|
||||
# The policy above grants paths under `auth/cert/`, and nothing
|
||||
# in this tree creates that mount. Without this, the grant names
|
||||
# a location that does not exist and every certificate login
|
||||
# fails — the controller's own, and the per-hive ones it later
|
||||
# issues against the same mount.
|
||||
#
|
||||
# Asked rather than attempted: `auth enable` errors on a mount
|
||||
# that already exists, and recognising that would tie a rebuild
|
||||
# to an error string we have never seen this store emit.
|
||||
mounted="$(bao auth list -format=json)"
|
||||
case "$mounted" in
|
||||
*'"cert/"'*) ;;
|
||||
*) bao auth enable cert ;;
|
||||
esac
|
||||
|
||||
# `certificate=` is the CA, so this role trusts every leaf that
|
||||
# CA signed and `allowed_common_names` is the whole of what
|
||||
# narrows it to one identity. ⚠️ The same CA signs each hive's
|
||||
# reader leaf with CN = the hive's name, so a hive named
|
||||
# `${controllerCn}` would satisfy this role.
|
||||
#
|
||||
# Named outside the `hive-*` namespace the policy grants, so the
|
||||
# controller cannot rewrite the role that constrains it.
|
||||
bao write auth/cert/certs/${lib.escapeShellArg controllerPolicyName} \
|
||||
certificate=@${tlsDir}/client-ca.pem \
|
||||
allowed_common_names=${lib.escapeShellArg controllerCn} \
|
||||
token_policies=${lib.escapeShellArg controllerPolicyName} \
|
||||
display_name=${lib.escapeShellArg controllerCn}
|
||||
'';
|
||||
};
|
||||
|
||||
containers.${cfg.machine} = {
|
||||
autoStart = true;
|
||||
ephemeral = false;
|
||||
|
|
@ -1158,6 +1076,78 @@ in
|
|||
# Restarting is an operator action with an unseal on the far side of
|
||||
# it, which is why nothing here tries to be clever about it.
|
||||
|
||||
# The swarm's first grant, written from in here because this is
|
||||
# where the store is reachable without a client certificate — which
|
||||
# is the point, since no role exists yet to issue one against.
|
||||
systemd.services.swarm-bao-controller-policy = lib.mkIf haveBootstrapToken {
|
||||
description = "write the swarm controller's bao policy and cert-auth role";
|
||||
after = [ "openbao.service" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
path = [
|
||||
baoDeploy.package
|
||||
pkgs.coreutils
|
||||
];
|
||||
# Named but not placed is a legitimate state: all-local supplies
|
||||
# the path as a default and the operator drops the file there
|
||||
# after `bao operator init`. Skipping rather than failing is also
|
||||
# what makes deleting the token at the end of that procedure safe.
|
||||
unitConfig.ConditionPathExists = baoDeploy.bootstrapTokenFile;
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
# A store that is up is not necessarily unsealed — under
|
||||
# `seal = "shamir"` an operator unseals by hand after every
|
||||
# restart — so early attempts legitimately fail. Bounded,
|
||||
# because a token that is wrong rather than early would
|
||||
# otherwise retry forever.
|
||||
Restart = "on-failure";
|
||||
RestartSec = 30;
|
||||
StartLimitBurst = 10;
|
||||
};
|
||||
environment.BAO_ADDR = "https://${cfg.domain}:${toString cfg.port}";
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
BAO_TOKEN="$(cat ${lib.escapeShellArg baoDeploy.bootstrapTokenFile})"
|
||||
export BAO_TOKEN
|
||||
|
||||
# Idempotent on purpose: a rebuild re-asserts the policy rather
|
||||
# than failing on one that already exists.
|
||||
printf '%s' ${lib.escapeShellArg controllerPolicyText} |
|
||||
bao policy write ${lib.escapeShellArg controllerPolicyName} -
|
||||
''
|
||||
+ lib.optionalString (baoDeploy.clientCaFile != null) ''
|
||||
|
||||
# The policy above grants paths under `auth/cert/`, and nothing
|
||||
# in this tree creates that mount. Without this, the grant names
|
||||
# a location that does not exist and every certificate login
|
||||
# fails — the controller's own, and the per-hive ones it later
|
||||
# issues against the same mount.
|
||||
#
|
||||
# Asked rather than attempted: `auth enable` errors on a mount
|
||||
# that already exists, and recognising that would tie a rebuild
|
||||
# to an error string we have never seen this store emit.
|
||||
mounted="$(bao auth list -format=json)"
|
||||
case "$mounted" in
|
||||
*'"cert/"'*) ;;
|
||||
*) bao auth enable cert ;;
|
||||
esac
|
||||
|
||||
# `certificate=` is the CA, so this role trusts every leaf that
|
||||
# CA signed and `allowed_common_names` is the whole of what
|
||||
# narrows it to one identity. ⚠️ The same CA signs each hive's
|
||||
# reader leaf with CN = the hive's name, so a hive named
|
||||
# `${controllerCn}` would satisfy this role.
|
||||
#
|
||||
# Named outside the `hive-*` namespace the policy grants, so the
|
||||
# controller cannot rewrite the role that constrains it.
|
||||
bao write auth/cert/certs/${lib.escapeShellArg controllerPolicyName} \
|
||||
certificate=@${tlsDir}/client-ca.pem \
|
||||
allowed_common_names=${lib.escapeShellArg controllerCn} \
|
||||
token_policies=${lib.escapeShellArg controllerPolicyName} \
|
||||
display_name=${lib.escapeShellArg controllerCn}
|
||||
'';
|
||||
};
|
||||
};
|
||||
};
|
||||
})
|
||||
|
|
|
|||
|
|
@ -525,40 +525,17 @@ let
|
|||
otelRemoteAuthelia.containers.swarm-otel.config.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential;
|
||||
}
|
||||
{
|
||||
# Reads the rendered unit on the HOST, which is where the write happens:
|
||||
# every API listener demands a client certificate, and the host is the
|
||||
# side that has one.
|
||||
name = "a store host with a placed bootstrap token renders the granting unit on the host";
|
||||
# Reads the rendered unit INSIDE the container, which is where the write
|
||||
# happens: reaching the store locally is what lets the grant be written
|
||||
# without a client certificate at all.
|
||||
name = "a store host with a placed bootstrap token renders the granting unit inside the container";
|
||||
ok =
|
||||
let
|
||||
u = baoGrantHere.systemd.services.swarm-bao-controller-policy;
|
||||
u = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy;
|
||||
in
|
||||
lib.hasInfix "/run/secrets/bao-bootstrap.token" u.script
|
||||
&& u.unitConfig.ConditionPathExists == "/run/secrets/bao-bootstrap.token";
|
||||
}
|
||||
{
|
||||
# The move is the fix, so pin the side it landed on: in the container it
|
||||
# had no identity to open a connection with, and no address that resolved
|
||||
# to the store from its own netns.
|
||||
name = "the granting unit is not rendered inside the store's container";
|
||||
ok = !(baoGrantHere.containers.swarm-bao.config.systemd.services ? swarm-bao-controller-policy);
|
||||
}
|
||||
{
|
||||
# `StartLimit*` are `[Unit]` settings that systemd ignores under
|
||||
# `[Service]`, so a bound written into `serviceConfig` renders, deploys
|
||||
# and does nothing. Asserted where nixpkgs puts it rather than where it
|
||||
# was written. The values are pinned because they are the bound: under
|
||||
# `shamir` a human unseals by hand, and anything shorter than a day gives
|
||||
# up first — `start-limit-hit` does not self-heal.
|
||||
name = "the granting unit's start limit lands in [Unit], not [Service]";
|
||||
ok =
|
||||
let
|
||||
u = baoGrantHere.systemd.services.swarm-bao-controller-policy;
|
||||
in
|
||||
toString u.unitConfig.StartLimitBurst == "2880"
|
||||
&& toString u.unitConfig.StartLimitIntervalSec == "90000"
|
||||
&& !(u.serviceConfig ? StartLimitBurst);
|
||||
}
|
||||
{
|
||||
# The grants themselves, and the `hive-` prefix is the whole point:
|
||||
# without it the controller can rewrite the policy that constrains it,
|
||||
|
|
@ -568,7 +545,7 @@ let
|
|||
name = "the controller's bao grants cannot reach the policy that constrains it";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
|
||||
s = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script;
|
||||
in
|
||||
lib.hasInfix "sys/policies/acl/hive-*" s && !(lib.hasInfix "sys/policies/acl/*" s);
|
||||
}
|
||||
|
|
@ -582,7 +559,7 @@ let
|
|||
name = "the controller may write agent credentials, and only under the agent prefix";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
|
||||
s = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script;
|
||||
in
|
||||
lib.hasInfix "secret/data/swarm/agents/*" s
|
||||
&& !(lib.hasInfix "secret/data/*" s)
|
||||
|
|
@ -596,7 +573,7 @@ let
|
|||
name = "the controller's grant on agent credentials is write-only";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
|
||||
s = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script;
|
||||
in
|
||||
lib.hasInfix "path \"secret/data/swarm/agents/*\" {\n capabilities = [\"create\", \"update\"]" s;
|
||||
}
|
||||
|
|
@ -607,7 +584,7 @@ let
|
|||
name = "the granting unit creates the cert auth mount and the controller's role";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
|
||||
s = baoGrantHere.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script;
|
||||
in
|
||||
lib.hasInfix "bao auth enable cert" s
|
||||
&& lib.hasInfix "auth/cert/certs/swarm-controller" s
|
||||
|
|
@ -760,7 +737,7 @@ let
|
|||
ok =
|
||||
let
|
||||
m = baoControllerHere;
|
||||
role = m.systemd.services.swarm-bao-controller-policy.script;
|
||||
role = m.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script;
|
||||
pki = m.systemd.services.swarm-bao-pki.script;
|
||||
in
|
||||
lib.hasInfix "cn-marker-not-a-default" role && lib.hasInfix "cn-marker-not-a-default" pki;
|
||||
|
|
@ -776,7 +753,8 @@ let
|
|||
name = "with no client CA the unit still writes the policy and skips the role";
|
||||
ok =
|
||||
let
|
||||
s = baoGrantNoClientCa.systemd.services.swarm-bao-controller-policy.script;
|
||||
s =
|
||||
baoGrantNoClientCa.containers.swarm-bao.config.systemd.services.swarm-bao-controller-policy.script;
|
||||
in
|
||||
lib.hasInfix "bao policy write" s
|
||||
&& !(lib.hasInfix "bao auth enable cert" s)
|
||||
|
|
|
|||
Loading…
Reference in a new issue