swarm-bao: write every swarm-* grant as a bao granter, not with a 24h token
Every unit that writes a bao policy or cert-auth role ran only while the operator-placed bootstrap token existed, and skipped silently otherwise. The token lives 24h, so on any real swarm a PR adding or changing a grant deployed with its unit skipped, and each one needed a manual token refresh (plus a root `bao policy write` when it added a path). A `bao-granter` principal now writes them. Its leaf is minted by swarm-bao-pki on the store host (0600 root, never copied off it), and its policy covers `swarm-*` policies, `swarm-*` cert-auth roles and `pki/roles/swarm-*` by glob, plus the mount and services-root paths the controller's unit already used. All ten granting units (controller, secret-publisher, matrix-ctl, matrix-token, queue-agent, grafana-oidc, otel-oidc, forwarder-oidc, services-issuer, nats-tls) log in with it instead of reading the token. They keep the 2880 x 30s retry, now require swarm-bao-pki, and when the store refuses the granter they fail and print the one-time step instead of skipping. swarm-bao-granter-role is the one unit left on the token. It enables the auth mounts (moved out of the controller's unit) and writes the granter's own policy and role. The bootstrap policy is renamed `bao-bootstrap` and shrinks to those five stanzas; it is shipped at /etc/hyperhive/bao-bootstrap-policy.hcl. The old name `swarm-bootstrap` matched the granter's own `swarm-*` glob. The granter's CN joins certAuthCns, so no hive can be named into its role. An assertion keeps both pki role names under `swarm-`. With no client CA the granting units no longer render, and a warning says so. module-eval pins the granter's policy stanza by stanza, what it cannot reach, that every call a granting unit makes is granted, and that only swarm-bao-granter-role reads the token. Refs #4704
This commit is contained in:
atlas
mara
parent
19cc1b12e2
commit
e9cec0da21
12 changed files with 898 additions and 443 deletions
37
nix/host-modules/bao-bootstrap-policy.hcl
Normal file
37
nix/host-modules/bao-bootstrap-policy.hcl
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
# The `bao-bootstrap` policy: what the 24h bootstrap token may do, and nothing
|
||||
# else. ../../docs/getting-started/setup.md has the operator write it with the
|
||||
# root token, from the copy ./swarm-bao.nix ships at
|
||||
# /etc/hyperhive/bao-bootstrap-policy.hcl; `swarm-bao-granter-role` then acts
|
||||
# with it. Every other grant is written by the `bao-granter` principal this
|
||||
# creates.
|
||||
#
|
||||
# Named outside `swarm-*`, so the granter cannot rewrite the policy the next
|
||||
# bootstrap token carries.
|
||||
#
|
||||
# Each stanza was derived with `bao <cmd> -output-policy`, which prints what a
|
||||
# command requires without sending it. ../module-eval/bao-grants.nix reads
|
||||
# this file and fails when the unit that uses the token calls a path it does
|
||||
# not grant.
|
||||
|
||||
# The auth mounts. Reading `sys/auth` is how the unit checks, and `sudo` is
|
||||
# what enabling one costs.
|
||||
path "sys/auth" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "sys/auth/cert" {
|
||||
capabilities = ["create", "update", "sudo"]
|
||||
}
|
||||
|
||||
path "sys/auth/approle" {
|
||||
capabilities = ["create", "update", "sudo"]
|
||||
}
|
||||
|
||||
# The granter's own policy and role, and nothing it may write.
|
||||
path "sys/policies/acl/bao-granter" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
path "auth/cert/certs/bao-granter" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
|
@ -12,9 +12,10 @@
|
|||
# with no ExecStart, so each gate below restates the one the reader's own
|
||||
# module puts on it. A reader whose gate changes must change here too.
|
||||
#
|
||||
# Ordering, never a requirement: a policy unit skips once the bootstrap token
|
||||
# is gone, and a skipped unit counts as done. `wants` as well as `after`, so a
|
||||
# reader started on its own pulls its policy unit into the same transaction.
|
||||
# Ordering, never a requirement: a policy unit that failed still counts as
|
||||
# done, and the reader's own retries carry it past that. `wants` as well as
|
||||
# `after`, so a reader started on its own pulls its policy unit into the same
|
||||
# transaction.
|
||||
{
|
||||
lib,
|
||||
config,
|
||||
|
|
|
|||
|
|
@ -108,6 +108,12 @@ in
|
|||
# on `client.pem`.
|
||||
forwarderOidcClientCertFile = lib.mkDefault "${pkiDir}/forwarder-oidc.pem";
|
||||
forwarderOidcClientKeyFile = lib.mkDefault "${pkiDir}/forwarder-oidc-key.pem";
|
||||
|
||||
# The granter: every `swarm-bao-*-policy` unit on this host logs in with
|
||||
# it. ⚠️ Unlike every other leaf here, never the file an operator copies:
|
||||
# its policy is root-equivalent and its only reader is this host.
|
||||
granterClientCertFile = lib.mkDefault "${pkiDir}/granter.pem";
|
||||
granterClientKeyFile = lib.mkDefault "${pkiDir}/granter-key.pem";
|
||||
};
|
||||
|
||||
# Idempotent on ABSENCE, never on content. Re-issuing the CA invalidates
|
||||
|
|
@ -248,6 +254,12 @@ in
|
|||
# the file an operator copies.
|
||||
[ -s ${pkiDir}/nats.pem ] || ${signLeaf} ${pkiDir} nats \
|
||||
${lib.escapeShellArg deployCfg.bao.natsCommonName} "" clientAuth
|
||||
|
||||
# The granter's, which writes every `swarm-*` grant. Minted here because
|
||||
# it opens the store for the units that create the roles every other
|
||||
# leaf logs in with. Stays on this host; see its default above.
|
||||
[ -s ${pkiDir}/granter.pem ] || ${signLeaf} ${pkiDir} granter \
|
||||
${lib.escapeShellArg deployCfg.bao.granterCommonName} "" clientAuth
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,159 +0,0 @@
|
|||
# The `swarm-bootstrap` policy: what the 24h bootstrap token may do, and
|
||||
# nothing else. ../../docs/getting-started/setup.md has the operator write it
|
||||
# with the root token; ./swarm-bao.nix's granting units then act with it.
|
||||
#
|
||||
# Each stanza was derived with `bao <cmd> -output-policy`, which prints what a
|
||||
# command requires without sending it. ../module-eval/bao-grants.nix reads
|
||||
# this file and fails when a unit that uses the token calls a path it does not
|
||||
# grant. The pki paths assume the default `servicesPkiMountPath` (`pki`),
|
||||
# `servicesPkiRoleName` (`swarm-services`) and `natsPkiRoleName` (`swarm-nats`).
|
||||
|
||||
# swarm-bao-controller-policy: the controller's own policy and role.
|
||||
path "sys/policies/acl/swarm-controller" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
path "auth/cert/certs/swarm-controller" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
# The auth mounts it creates. Reading `sys/auth` is how the unit checks, and
|
||||
# `sudo` is what enabling one costs.
|
||||
path "sys/auth" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "sys/auth/cert" {
|
||||
capabilities = ["create", "update", "sudo"]
|
||||
}
|
||||
|
||||
path "sys/auth/approle" {
|
||||
capabilities = ["create", "update", "sudo"]
|
||||
}
|
||||
|
||||
# The KV and PKI engines, checked the same way. Enabling a secrets engine does
|
||||
# not ask for `sudo`.
|
||||
path "sys/mounts" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "sys/mounts/secret" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
path "sys/mounts/pki" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
path "sys/mounts/pki/tune" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
# The services root: generated once, read back on every run, and replaced
|
||||
# only when it can no longer outlive a leaf.
|
||||
path "pki/issuers" {
|
||||
capabilities = ["list"]
|
||||
}
|
||||
|
||||
path "pki/cert/ca" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "pki/root" {
|
||||
capabilities = ["delete", "sudo"]
|
||||
}
|
||||
|
||||
path "pki/root/generate/internal" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
path "pki/roles/swarm-services" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
# swarm-bao-secret-publisher-policy
|
||||
path "sys/policies/acl/swarm-secret-publisher" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
path "auth/cert/certs/swarm-secret-publisher" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
# swarm-bao-matrix-ctl-policy
|
||||
path "sys/policies/acl/swarm-matrix-ctl" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
path "auth/cert/certs/swarm-matrix-ctl" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
# swarm-bao-services-issuer-policy
|
||||
path "sys/policies/acl/swarm-services-issuer" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
path "auth/cert/certs/swarm-services-issuer" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
# swarm-bao-grafana-oidc-policy
|
||||
path "sys/policies/acl/swarm-grafana-oidc" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
path "auth/cert/certs/swarm-grafana-oidc" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
# swarm-bao-otel-oidc-policy
|
||||
path "sys/policies/acl/swarm-otel-oidc" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
path "auth/cert/certs/swarm-otel-oidc" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
# swarm-bao-forwarder-oidc-policy
|
||||
path "sys/policies/acl/swarm-forwarder-oidc" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
path "auth/cert/certs/swarm-forwarder-oidc" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
# swarm-bao-nats-tls-policy: the queue's own pki role, beside
|
||||
# `swarm-services` above, and its policy and login role.
|
||||
path "pki/roles/swarm-nats" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
path "sys/policies/acl/swarm-nats" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
path "auth/cert/certs/swarm-nats" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
# swarm-bao-matrix-token-policy and swarm-bao-queue-agent-policy write one
|
||||
# policy and role per hive, `<prefix>-<hive>`, so these two are globs. Each
|
||||
# stops at its own prefix.
|
||||
path "sys/policies/acl/swarm-matrix-token-*" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
path "auth/cert/certs/swarm-matrix-token-*" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
path "sys/policies/acl/swarm-queue-agent-*" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
path "auth/cert/certs/swarm-queue-agent-*" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
|
@ -151,7 +151,7 @@ let
|
|||
# rather than as the missing setting it is.
|
||||
haveServerTls = baoDeploy.serverCertFile != null && baoDeploy.serverKeyFile != null;
|
||||
|
||||
# The credential that writes the swarm's first grant. A token and not a
|
||||
# The credential that writes the granter's role below. A token and not a
|
||||
# certificate: cert auth answers a *role*, so nothing can authenticate here
|
||||
# until some role exists, and whatever creates the first one cannot itself
|
||||
# use one. An operator places it — ../../docs/getting-started/setup.md.
|
||||
|
|
@ -163,6 +163,137 @@ let
|
|||
bootstrapTokenDir =
|
||||
if haveBootstrapToken then builtins.dirOf baoDeploy.bootstrapTokenFile else null;
|
||||
|
||||
# The principal every `swarm-bao-*-policy` unit logs in as, so a new or
|
||||
# changed `swarm-*` grant applies on deploy with no operator step. Policy and
|
||||
# role share one name, outside both `swarm-*` and `hive-*`: neither the
|
||||
# granter's globs nor the controller's reach the objects that constrain it.
|
||||
granterPolicyName = "bao-granter";
|
||||
granterCn = baoDeploy.granterCommonName;
|
||||
|
||||
# No CA means no login role can be written, so nothing could log in as the
|
||||
# granter; the units that need it do not render.
|
||||
haveGranter =
|
||||
baoDeploy.granterClientCertFile != null
|
||||
&& baoDeploy.granterClientKeyFile != null
|
||||
&& baoDeploy.clientCaFile != null;
|
||||
|
||||
# ⚠️ ROOT-EQUIVALENT BY CONSTRUCTION. No ACL constrains the body of a policy,
|
||||
# so a principal that may write `swarm-*` policies and the roles attaching
|
||||
# them may grant itself anything. What bounds it is that every policy it
|
||||
# writes is rendered from this file, and that its key never leaves this host.
|
||||
#
|
||||
# A trailing `*` in a bao ACL path is a pure string-prefix match, and an
|
||||
# exact path wins over any prefix.
|
||||
#
|
||||
# The first three are the per-principal grants. The rest are what
|
||||
# `swarm-bao-controller-policy` does besides grants: the KV and pki mounts and
|
||||
# the services root. No `sys/auth`: the auth mounts are created with the
|
||||
# bootstrap token by `swarm-bao-granter-role`.
|
||||
#
|
||||
# Piped as a shell-quoted argument like `controllerPolicyText`, so
|
||||
# ../module-eval/bao-grants.nix can read it out of the unit script.
|
||||
granterPolicyText = ''
|
||||
path "sys/policies/acl/swarm-*" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
path "auth/cert/certs/swarm-*" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
path "${servicesPkiMountPath}/roles/swarm-*" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
path "sys/mounts" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "sys/mounts/${credentialMountPath}" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
path "sys/mounts/${servicesPkiMountPath}" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
path "sys/mounts/${servicesPkiMountPath}/tune" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
|
||||
path "${servicesPkiMountPath}/issuers" {
|
||||
capabilities = ["list"]
|
||||
}
|
||||
|
||||
path "${servicesPkiMountPath}/cert/ca" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
|
||||
path "${servicesPkiMountPath}/root" {
|
||||
capabilities = ["delete", "sudo"]
|
||||
}
|
||||
|
||||
path "${servicesPkiMountPath}/root/generate/internal" {
|
||||
capabilities = ["create", "update"]
|
||||
}
|
||||
'';
|
||||
|
||||
# What a granting unit prints when the store refuses the granter: the
|
||||
# one-time step, runnable as root on this host.
|
||||
granterSetupSteps = [
|
||||
"read -rs BAO_TOKEN && export BAO_TOKEN # the root token from 'bao operator init'"
|
||||
"bao policy write bao-bootstrap /etc/hyperhive/bao-bootstrap-policy.hcl"
|
||||
]
|
||||
++ (
|
||||
if haveBootstrapToken then
|
||||
[
|
||||
"bao token create -policy=bao-bootstrap -ttl=24h -orphan -display-name=bao-bootstrap -field=token | install -D -m 0600 /dev/stdin ${baoDeploy.bootstrapTokenFile}"
|
||||
"unset BAO_TOKEN"
|
||||
"systemctl restart swarm-bao-granter-role"
|
||||
]
|
||||
else
|
||||
[
|
||||
"# then set services.hyperhive.deploy.bao.bootstrapTokenFile on this host, deploy, and place a token there:"
|
||||
"bao token create -policy=bao-bootstrap -ttl=24h -orphan -display-name=bao-bootstrap -field=token"
|
||||
]
|
||||
);
|
||||
|
||||
# The login every granting unit starts with. It FAILS rather than skips: a
|
||||
# grant that was not written is otherwise invisible until whatever needs it
|
||||
# fails somewhere else. `bao status` exits 0 only on a reachable, unsealed
|
||||
# store, which separates "the granter is not set up" from "retry later";
|
||||
# either way bao's own message follows.
|
||||
granterLogin = ''
|
||||
err="$(mktemp)"
|
||||
trap 'rm -f "$err"' EXIT
|
||||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||||
if bao status >/dev/null 2>&1; then
|
||||
echo ${lib.escapeShellArg "the store is unsealed but refused the granter's certificate (CN ${granterCn}): the ${granterPolicyName} role is not set up."} >&2
|
||||
echo "one-time step, as root on this host (docs/getting-started/setup.md):" >&2
|
||||
${lib.concatMapStringsSep "\n" (l: "echo ${lib.escapeShellArg " ${l}"} >&2") granterSetupSteps}
|
||||
else
|
||||
echo "the store is sealed or unreachable; retrying." >&2
|
||||
fi
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
export BAO_TOKEN
|
||||
'';
|
||||
|
||||
# The granter's certificate for the granting units. The `baoCli` wrapper
|
||||
# only defaults these, so the unit's environment wins.
|
||||
granterEnv = {
|
||||
BAO_CLIENT_CERT = baoDeploy.granterClientCertFile;
|
||||
BAO_CLIENT_KEY = baoDeploy.granterClientKeyFile;
|
||||
};
|
||||
|
||||
# `swarm-bao-pki` mints the granter's leaf; the granter's role is written by
|
||||
# `swarm-bao-granter-role`, which normally skips, hence ordering only there.
|
||||
granterAfter = [
|
||||
"swarm-bao-pki.service"
|
||||
"swarm-bao-granter-role.service"
|
||||
];
|
||||
|
||||
# The name both ends must agree on: the cert-auth role below attaches this
|
||||
# policy by spelling it the same way, and is itself named after it.
|
||||
controllerPolicyName = "swarm-controller";
|
||||
|
|
@ -546,27 +677,25 @@ let
|
|||
# after it.
|
||||
#
|
||||
# `after` and not `requires`, for the reason the publisher's unit states: the
|
||||
# controller's unit creates the KV and cert-auth mounts this one writes into,
|
||||
# but a failed oneshot still counts as finished, so ordering plus this unit's
|
||||
# own retry is what converges.
|
||||
#
|
||||
# The role write is inside the client-CA branch and the policy write is not,
|
||||
# exactly as the three above: with no CA there is no trust anchor for a login
|
||||
# role, but the policy it would attach is still worth asserting.
|
||||
# controller's and the granter's units create the mounts this one writes
|
||||
# into, but a failed oneshot still counts as finished, so ordering plus this
|
||||
# unit's own retry is what converges.
|
||||
readerPolicyUnit =
|
||||
description: objects:
|
||||
lib.mkIf haveBootstrapToken {
|
||||
lib.mkIf haveGranter {
|
||||
inherit description;
|
||||
after = [
|
||||
"container@${cfg.machine}.service"
|
||||
"swarm-bao-controller-policy.service"
|
||||
];
|
||||
]
|
||||
++ granterAfter;
|
||||
requires = [ "swarm-bao-pki.service" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
path = [
|
||||
baoCli
|
||||
pkgs.coreutils
|
||||
];
|
||||
unitConfig.ConditionPathExists = baoDeploy.bootstrapTokenFile;
|
||||
environment = granterEnv;
|
||||
# Same unseal wait as its siblings above, for the reason stated there:
|
||||
# under `seal = "shamir"` a human unseals by hand.
|
||||
startLimitBurst = 2880;
|
||||
|
|
@ -580,14 +709,11 @@ let
|
|||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
BAO_TOKEN="$(cat ${lib.escapeShellArg baoDeploy.bootstrapTokenFile})"
|
||||
export BAO_TOKEN
|
||||
|
||||
${granterLogin}
|
||||
''
|
||||
+ lib.concatMapStrings readerPolicyWrite objects
|
||||
+ lib.optionalString (baoDeploy.clientCaFile != null) (
|
||||
"\n" + lib.concatMapStrings readerRoleWrite objects
|
||||
);
|
||||
+ "\n"
|
||||
+ lib.concatMapStrings readerRoleWrite objects;
|
||||
};
|
||||
|
||||
# Every listener serves the same identity: they differ in which address
|
||||
|
|
@ -979,19 +1105,22 @@ in
|
|||
default = null;
|
||||
example = "/var/lib/swarm-bao-bootstrap/grant.token";
|
||||
description = ''
|
||||
Token used **once per swarm** to write the first authorisation grants,
|
||||
after which every client authenticates with a certificate instead.
|
||||
Token used **once per swarm** to create the store's cert-auth mount and
|
||||
the `bao-granter` policy and role, after which every granting unit
|
||||
logs in as the granter with a certificate instead.
|
||||
|
||||
Cert auth answers a *role*, so no client can authenticate until some
|
||||
role exists — and creating that first one is what this token is for.
|
||||
role exists — and creating the granter's is what this token is for.
|
||||
It has to come from outside that cycle, which is why an operator places
|
||||
it rather than the deployment minting it.
|
||||
|
||||
Produce it from the root token `bao operator init` printed, scoped to
|
||||
that one policy write and nothing else, then delete it once the swarm
|
||||
has come up — {file}`docs/getting-started/setup.md` has the commands.
|
||||
Setting this is what enables the granting unit; leaving it null means
|
||||
the deployment writes those grants some other way.
|
||||
Produce it from the root token `bao operator init` printed, under the
|
||||
`bao-bootstrap` policy shipped at
|
||||
{file}`/etc/hyperhive/bao-bootstrap-policy.hcl`, then delete it once
|
||||
`swarm-bao-granter-role` has run —
|
||||
{file}`docs/getting-started/setup.md` has the commands. Setting this is
|
||||
what renders `swarm-bao-granter-role`; while it is null, a store whose
|
||||
granter is not set up has no way to set it up.
|
||||
|
||||
A path, never a value.
|
||||
'';
|
||||
|
|
@ -1420,6 +1549,48 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
granterCommonName = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "bao-granter";
|
||||
description = ''
|
||||
Subject the store's `bao-granter` cert-auth role accepts: the identity
|
||||
every `swarm-bao-*-policy` unit on the store's host logs in as to write
|
||||
the `swarm-*` policies, cert-auth roles and pki roles.
|
||||
|
||||
⚠️ Root-equivalent: it may write a `swarm-*` policy with any content.
|
||||
Reserved as a hive name by ./swarm.nix, like its siblings.
|
||||
'';
|
||||
};
|
||||
|
||||
granterClientCertFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "/var/lib/swarm-bao-pki/granter.pem";
|
||||
description = ''
|
||||
Certificate the store's granting units present to the store. Its
|
||||
subject must be
|
||||
{option}`services.hyperhive.deploy.bao.granterCommonName`.
|
||||
|
||||
Null, or a null
|
||||
{option}`services.hyperhive.deploy.bao.clientCaFile`, means this
|
||||
deployment writes those grants some other way: no granting unit
|
||||
renders.
|
||||
|
||||
⚠️ Unlike every other leaf the store's host mints, this one is never
|
||||
copied to another host; its only reader is that host.
|
||||
'';
|
||||
};
|
||||
|
||||
granterClientKeyFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "/var/lib/swarm-bao-pki/granter-key.pem";
|
||||
description = ''
|
||||
Private key for
|
||||
{option}`services.hyperhive.deploy.bao.granterClientCertFile`.
|
||||
'';
|
||||
};
|
||||
|
||||
serverCaFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
|
|
@ -1635,8 +1806,29 @@ in
|
|||
grant reads every secret in the store; this role reads one path.
|
||||
'';
|
||||
}
|
||||
{
|
||||
# The granter writes pki roles through `roles/swarm-*` and nothing
|
||||
# else, so a role named otherwise is a 403 at deploy time.
|
||||
assertion =
|
||||
!haveGranter
|
||||
|| (lib.hasPrefix "swarm-" servicesPkiRoleName && lib.hasPrefix "swarm-" natsPkiRoleName);
|
||||
message = ''
|
||||
services.hyperhive.deploy.bao.servicesPkiRoleName
|
||||
(${servicesPkiRoleName}) and
|
||||
services.hyperhive.deploy.bao.natsPkiRoleName (${natsPkiRoleName})
|
||||
must both start with `swarm-`: the bao granter that writes them may
|
||||
write pki roles under that prefix only.
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
warnings = lib.optional (haveServerTls && baoDeploy.clientCaFile == null) ''
|
||||
services.hyperhive.deploy.bao.clientCaFile is null, so no cert-auth
|
||||
role can be written and no client can log in to the swarm secret store.
|
||||
None of the swarm-bao-*-policy units render: this deployment writes no
|
||||
bao policy or role.
|
||||
'';
|
||||
|
||||
# The name every reader dials, made resolvable where the store runs.
|
||||
# Cross-hive traffic always goes via the domain; only what it resolves
|
||||
# to varies, and a multi-host swarm is the operator's upstream DNS. This
|
||||
|
|
@ -1664,6 +1856,10 @@ in
|
|||
# addresses on every command.
|
||||
environment.systemPackages = [ baoCli ];
|
||||
|
||||
# The policy the operator writes with the root token for the one-time
|
||||
# granter step, on the host where that step runs.
|
||||
environment.etc."hyperhive/bao-bootstrap-policy.hcl".source = ./bao-bootstrap-policy.hcl;
|
||||
|
||||
# The in-container unit plus the host-side ones this module defines.
|
||||
# `swarm-bao-pki` and `swarm-bao-matrix-token` are declared by the glue
|
||||
# modules that create them, per the option's own rule — and a name
|
||||
|
|
@ -1674,6 +1870,7 @@ in
|
|||
"swarm-bao-certs"
|
||||
"swarm-bao-token"
|
||||
"swarm-bao-forwarder-oidc"
|
||||
"swarm-bao-granter-role"
|
||||
"swarm-bao-controller-policy"
|
||||
"swarm-bao-secret-publisher-policy"
|
||||
"swarm-bao-matrix-ctl-policy"
|
||||
|
|
@ -1965,17 +2162,85 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
# The swarm's first grant, written from the HOST. Every API listener sets
|
||||
# `tls_require_and_verify_client_cert`, so a client needs an identity
|
||||
# wherever it runs — and only the host has one. The bootstrap token is a
|
||||
# host path too; the container saw it through a bind mount.
|
||||
systemd.services.swarm-bao-controller-policy = lib.mkIf haveBootstrapToken {
|
||||
description = "write the swarm controller's bao policy and cert-auth role";
|
||||
# The one unit that still acts with the bootstrap token: it creates the
|
||||
# auth mounts and the granter's own policy and role, which nothing the
|
||||
# granter holds may write. Skipped while the token is absent, which is
|
||||
# the steady state once it has run; the granting units below are the ones
|
||||
# that fail loudly when it has never run.
|
||||
#
|
||||
# Ordering only toward them, never a requirement, for that same reason.
|
||||
systemd.services.swarm-bao-granter-role = lib.mkIf (haveBootstrapToken && haveGranter) {
|
||||
description = "write the bao granter's policy and cert-auth role with the bootstrap token";
|
||||
after = [ "container@${cfg.machine}.service" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
# The wrapper rather than the package: it carries the address, the CA
|
||||
# and this host's certificate, which is what makes running here cheaper
|
||||
# than shipping an identity the other way.
|
||||
path = [
|
||||
baoCli
|
||||
pkgs.coreutils
|
||||
];
|
||||
# Named but not placed is a legitimate state: all-local supplies the
|
||||
# path as a default and the operator drops the file there after
|
||||
# `bao operator init`. Skipping rather than failing is also what makes
|
||||
# deleting the token at the end of that procedure safe.
|
||||
unitConfig.ConditionPathExists = baoDeploy.bootstrapTokenFile;
|
||||
# Same unseal wait as the controller's unit below, for the reason
|
||||
# stated there.
|
||||
startLimitBurst = 2880;
|
||||
startLimitIntervalSec = 90000;
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
Restart = "on-failure";
|
||||
RestartSec = 30;
|
||||
};
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
BAO_TOKEN="$(cat ${lib.escapeShellArg baoDeploy.bootstrapTokenFile})"
|
||||
export BAO_TOKEN
|
||||
|
||||
# Every cert-auth role in this file lives under `auth/cert/`, and
|
||||
# nothing else creates that mount.
|
||||
#
|
||||
# Asked rather than attempted: `auth enable` errors on a mount
|
||||
# that already exists, and recognising that would tie a rebuild
|
||||
# to an error string we have never seen this store emit.
|
||||
mounted="$(bao auth list -format=json)"
|
||||
case "$mounted" in
|
||||
*'"cert/"'*) ;;
|
||||
*) bao auth enable cert ;;
|
||||
esac
|
||||
|
||||
case "$mounted" in
|
||||
*'"approle/"'*) ;;
|
||||
*) bao auth enable approle ;;
|
||||
esac
|
||||
|
||||
printf '%s' ${lib.escapeShellArg granterPolicyText} |
|
||||
bao policy write ${lib.escapeShellArg granterPolicyName} -
|
||||
|
||||
# The TTL bounds a leaked login token to minutes; the leaf is what
|
||||
# lives long.
|
||||
bao write auth/cert/certs/${lib.escapeShellArg granterPolicyName} \
|
||||
certificate=@${tlsDir}/client-ca.pem \
|
||||
allowed_common_names=${lib.escapeShellArg granterCn} \
|
||||
token_policies=${lib.escapeShellArg granterPolicyName} \
|
||||
display_name=${lib.escapeShellArg granterCn} \
|
||||
token_ttl=15m \
|
||||
token_max_ttl=15m
|
||||
'';
|
||||
};
|
||||
|
||||
# The swarm's first grant, written from the HOST. Every API listener sets
|
||||
# `tls_require_and_verify_client_cert`, so a client needs an identity
|
||||
# wherever it runs — and only the host has one: the granter's leaf.
|
||||
systemd.services.swarm-bao-controller-policy = lib.mkIf haveGranter {
|
||||
description = "write the swarm controller's bao policy and cert-auth role";
|
||||
after = [ "container@${cfg.machine}.service" ] ++ granterAfter;
|
||||
requires = [ "swarm-bao-pki.service" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
# The wrapper rather than the package: it carries the address and the
|
||||
# CA, which is what makes running here cheaper than shipping an
|
||||
# identity the other way.
|
||||
path = [
|
||||
baoCli
|
||||
pkgs.coreutils
|
||||
|
|
@ -1983,11 +2248,7 @@ in
|
|||
# regeneration guard below turns that into a decision.
|
||||
pkgs.openssl
|
||||
];
|
||||
# Named but not placed is a legitimate state: all-local supplies the
|
||||
# path as a default and the operator drops the file there after
|
||||
# `bao operator init`. Skipping rather than failing is also what makes
|
||||
# deleting the token at the end of that procedure safe.
|
||||
unitConfig.ConditionPathExists = baoDeploy.bootstrapTokenFile;
|
||||
environment = granterEnv;
|
||||
# A store that is up is not necessarily unsealed — under
|
||||
# `seal = "shamir"` an operator unseals BY HAND, so early attempts fail
|
||||
# for as long as that takes, which can be a day.
|
||||
|
|
@ -2009,8 +2270,7 @@ in
|
|||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
BAO_TOKEN="$(cat ${lib.escapeShellArg baoDeploy.bootstrapTokenFile})"
|
||||
export BAO_TOKEN
|
||||
${granterLogin}
|
||||
|
||||
# Idempotent on purpose: a rebuild re-asserts the policy rather
|
||||
# than failing on one that already exists.
|
||||
|
|
@ -2023,11 +2283,9 @@ in
|
|||
# controller's first credential write fails against a grant that
|
||||
# reads as correct.
|
||||
#
|
||||
# Outside the client-CA block below on purpose: this mount is what
|
||||
# the controller writes *through*, independent of who may log in.
|
||||
#
|
||||
# Asked rather than attempted, same as the auth mount: `secrets
|
||||
# enable` errors on a path already in use.
|
||||
# Asked rather than attempted, same as the auth mounts in
|
||||
# `swarm-bao-granter-role`: `secrets enable` errors on a path
|
||||
# already in use.
|
||||
mounts="$(bao secrets list -format=json)"
|
||||
case "$mounts" in
|
||||
*'"${credentialMountPath}/"'*) ;;
|
||||
|
|
@ -2186,28 +2444,6 @@ in
|
|||
key_bits=4096 \
|
||||
ttl=${servicesPkiLeafTtl} \
|
||||
max_ttl=${servicesPkiLeafTtl}
|
||||
''
|
||||
+ lib.optionalString (baoDeploy.clientCaFile != null) ''
|
||||
|
||||
# The policy above grants paths under `auth/cert/`, and nothing
|
||||
# in this tree creates that mount. Without this, the grant names
|
||||
# a location that does not exist and every certificate login
|
||||
# fails — the controller's own, and the per-hive ones it later
|
||||
# issues against the same mount.
|
||||
#
|
||||
# Asked rather than attempted: `auth enable` errors on a mount
|
||||
# that already exists, and recognising that would tie a rebuild
|
||||
# to an error string we have never seen this store emit.
|
||||
mounted="$(bao auth list -format=json)"
|
||||
case "$mounted" in
|
||||
*'"cert/"'*) ;;
|
||||
*) bao auth enable cert ;;
|
||||
esac
|
||||
|
||||
case "$mounted" in
|
||||
*'"approle/"'*) ;;
|
||||
*) bao auth enable approle ;;
|
||||
esac
|
||||
|
||||
# `certificate=` is the CA, so this role trusts every leaf that
|
||||
# CA signed and `allowed_common_names` is the whole narrowing —
|
||||
|
|
@ -2230,23 +2466,25 @@ in
|
|||
# Widening it to two principals would make the name wrong, and renaming it
|
||||
# would make that instruction wrong.
|
||||
#
|
||||
# `after` and not `requires`: the unit above creates the KV and cert-auth
|
||||
# `after` and not `requires`: the unit above and the granter's create the
|
||||
# mounts this one writes into, but a failed oneshot still counts as
|
||||
# finished, so `requires` would neither wait for its success nor re-run
|
||||
# this one when its own retry eventually lands. Ordering plus this unit's
|
||||
# own retry is what actually converges.
|
||||
systemd.services.swarm-bao-secret-publisher-policy = lib.mkIf haveBootstrapToken {
|
||||
systemd.services.swarm-bao-secret-publisher-policy = lib.mkIf haveGranter {
|
||||
description = "write the swarm secret publisher's bao policy and cert-auth role";
|
||||
after = [
|
||||
"container@${cfg.machine}.service"
|
||||
"swarm-bao-controller-policy.service"
|
||||
];
|
||||
]
|
||||
++ granterAfter;
|
||||
requires = [ "swarm-bao-pki.service" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
path = [
|
||||
baoCli
|
||||
pkgs.coreutils
|
||||
];
|
||||
unitConfig.ConditionPathExists = baoDeploy.bootstrapTokenFile;
|
||||
environment = granterEnv;
|
||||
# Same unseal wait as its sibling above, for the reason stated there:
|
||||
# under `seal = "shamir"` a human unseals by hand, which can take a day.
|
||||
startLimitBurst = 2880;
|
||||
|
|
@ -2260,13 +2498,10 @@ in
|
|||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
BAO_TOKEN="$(cat ${lib.escapeShellArg baoDeploy.bootstrapTokenFile})"
|
||||
export BAO_TOKEN
|
||||
${granterLogin}
|
||||
|
||||
printf '%s' ${lib.escapeShellArg secretPublisherPolicyText} |
|
||||
bao policy write ${lib.escapeShellArg secretPublisherPolicyName} -
|
||||
''
|
||||
+ lib.optionalString (baoDeploy.clientCaFile != null) ''
|
||||
|
||||
bao write auth/cert/certs/${lib.escapeShellArg secretPublisherPolicyName} \
|
||||
certificate=@${tlsDir}/client-ca.pem \
|
||||
|
|
@ -2283,18 +2518,20 @@ in
|
|||
# creates the mounts this one writes into, but a failed oneshot still
|
||||
# counts as finished, so only ordering plus this unit's own retry
|
||||
# converges.
|
||||
systemd.services.swarm-bao-matrix-ctl-policy = lib.mkIf haveBootstrapToken {
|
||||
systemd.services.swarm-bao-matrix-ctl-policy = lib.mkIf haveGranter {
|
||||
description = "write swarm-matrix-ctl's bao policy and cert-auth role";
|
||||
after = [
|
||||
"container@${cfg.machine}.service"
|
||||
"swarm-bao-controller-policy.service"
|
||||
];
|
||||
]
|
||||
++ granterAfter;
|
||||
requires = [ "swarm-bao-pki.service" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
path = [
|
||||
baoCli
|
||||
pkgs.coreutils
|
||||
];
|
||||
unitConfig.ConditionPathExists = baoDeploy.bootstrapTokenFile;
|
||||
environment = granterEnv;
|
||||
# Same unseal wait as its two siblings above, for the reason stated
|
||||
# there: under `seal = "shamir"` a human unseals by hand.
|
||||
startLimitBurst = 2880;
|
||||
|
|
@ -2308,13 +2545,10 @@ in
|
|||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
BAO_TOKEN="$(cat ${lib.escapeShellArg baoDeploy.bootstrapTokenFile})"
|
||||
export BAO_TOKEN
|
||||
${granterLogin}
|
||||
|
||||
printf '%s' ${lib.escapeShellArg matrixCtlPolicyText} |
|
||||
bao policy write ${lib.escapeShellArg matrixCtlPolicyName} -
|
||||
''
|
||||
+ lib.optionalString (baoDeploy.clientCaFile != null) ''
|
||||
|
||||
bao write auth/cert/certs/${lib.escapeShellArg matrixCtlPolicyName} \
|
||||
certificate=@${tlsDir}/client-ca.pem \
|
||||
|
|
@ -2349,18 +2583,20 @@ in
|
|||
# The policy text moved here from the controller's unit, where it sat
|
||||
# while it attached to nothing — a policy and the role that carries it
|
||||
# belong in one place, and now there is a principal to put them with.
|
||||
systemd.services.swarm-bao-services-issuer-policy = lib.mkIf haveBootstrapToken {
|
||||
systemd.services.swarm-bao-services-issuer-policy = lib.mkIf haveGranter {
|
||||
description = "write the swarm services issuer's bao policy and cert-auth role";
|
||||
after = [
|
||||
"container@${cfg.machine}.service"
|
||||
"swarm-bao-controller-policy.service"
|
||||
];
|
||||
]
|
||||
++ granterAfter;
|
||||
requires = [ "swarm-bao-pki.service" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
path = [
|
||||
baoCli
|
||||
pkgs.coreutils
|
||||
];
|
||||
unitConfig.ConditionPathExists = baoDeploy.bootstrapTokenFile;
|
||||
environment = granterEnv;
|
||||
# Same unseal wait as its three siblings above, for the reason stated
|
||||
# there: under `seal = "shamir"` a human unseals by hand.
|
||||
startLimitBurst = 2880;
|
||||
|
|
@ -2374,13 +2610,10 @@ in
|
|||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
BAO_TOKEN="$(cat ${lib.escapeShellArg baoDeploy.bootstrapTokenFile})"
|
||||
export BAO_TOKEN
|
||||
${granterLogin}
|
||||
|
||||
printf '%s' ${lib.escapeShellArg servicesIssuerPolicyText} |
|
||||
bao policy write ${lib.escapeShellArg servicesIssuerPolicyName} -
|
||||
''
|
||||
+ lib.optionalString (baoDeploy.clientCaFile != null) ''
|
||||
|
||||
bao write auth/cert/certs/${lib.escapeShellArg servicesIssuerPolicyName} \
|
||||
certificate=@${tlsDir}/client-ca.pem \
|
||||
|
|
@ -2398,18 +2631,20 @@ in
|
|||
# The role narrows exactly as `swarm-services` does, to one name. It is
|
||||
# the queue's domain alone, since the same name reaches it from every
|
||||
# hive; no IP SANs, since nothing dials an address.
|
||||
systemd.services.swarm-bao-nats-tls-policy = lib.mkIf haveBootstrapToken {
|
||||
systemd.services.swarm-bao-nats-tls-policy = lib.mkIf haveGranter {
|
||||
description = "write the swarm queue's pki role, bao policy and cert-auth role";
|
||||
after = [
|
||||
"container@${cfg.machine}.service"
|
||||
"swarm-bao-controller-policy.service"
|
||||
];
|
||||
]
|
||||
++ granterAfter;
|
||||
requires = [ "swarm-bao-pki.service" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
path = [
|
||||
baoCli
|
||||
pkgs.coreutils
|
||||
];
|
||||
unitConfig.ConditionPathExists = baoDeploy.bootstrapTokenFile;
|
||||
environment = granterEnv;
|
||||
# Same unseal wait as its siblings above.
|
||||
startLimitBurst = 2880;
|
||||
startLimitIntervalSec = 90000;
|
||||
|
|
@ -2422,8 +2657,7 @@ in
|
|||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
BAO_TOKEN="$(cat ${lib.escapeShellArg baoDeploy.bootstrapTokenFile})"
|
||||
export BAO_TOKEN
|
||||
${granterLogin}
|
||||
|
||||
bao write ${lib.escapeShellArg "${servicesPkiMountPath}/roles/${natsPkiRoleName}"} \
|
||||
allowed_domains=${lib.escapeShellArg hyperhiveCfg.swarm.nats.domain} \
|
||||
|
|
@ -2443,8 +2677,6 @@ in
|
|||
|
||||
printf '%s' ${lib.escapeShellArg natsPolicyText} |
|
||||
bao policy write ${lib.escapeShellArg natsPolicyName} -
|
||||
''
|
||||
+ lib.optionalString (baoDeploy.clientCaFile != null) ''
|
||||
|
||||
bao write auth/cert/certs/${lib.escapeShellArg natsPolicyName} \
|
||||
certificate=@${tlsDir}/client-ca.pem \
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ let
|
|||
deployCfg.bao.forwarderOidcCommonName
|
||||
deployCfg.bao.servicesIssuerCommonName
|
||||
deployCfg.bao.natsCommonName
|
||||
deployCfg.bao.granterCommonName
|
||||
]
|
||||
# The two per-hive readers' subjects, spelled out per hive rather than as the
|
||||
# prefix. The prefix alone would reserve the wrong string: the role for hive
|
||||
|
|
|
|||
Loading…
Reference in a new issue