9f3f01450b
The state directory was bind-mounted from the host so a nixos-container destroy could not take the swarm's secrets with it. No sibling service does that -- swarm-grafana keeps its sqlite database inside the container on ephemeral = false -- and the bind is what broke the store: upstream pairs StateDirectory= with DynamicUser=, systemd relocates the state to /var/lib/private/openbao, and that rename fails EBUSY on an active mount point, so the unit died at STATE_DIRECTORY before bao ever ran. The TLS material still has to cross the boundary, because a host unit writes it and the container reads it, so it moves to its own small bind at /var/lib/swarm-bao-tls rather than riding along in the state directory. That directory is 0755 and read-only inside: the certificate and client CA are public and are read straight off the mount. The private key is not. install -m 0600 leaves it root-owned and the service runs as a DynamicUser, so the bind-mounted file is unreadable to it -- which the old layout hid, because StateDirectory chowned the whole tree on the way past. LoadCredential is systemd's mechanism for precisely this: PID 1 opens the source as root and re-exposes it inside the unit owned by the service's own account.
597 lines
26 KiB
Nix
597 lines
26 KiB
Nix
# The swarm's secret store: one OpenBao for the whole swarm, in a
|
||
# `swarm-bao` nixos-container.
|
||
#
|
||
# Today every credential in ./swarm-*.nix is minted where it is read or copied
|
||
# there by a delivery unit (docs/swarm/secrets.md), which ties each secret's
|
||
# lifetime to its container's.
|
||
#
|
||
# ⚠️ It authenticates hive clients with a CLIENT CERTIFICATE, not with the
|
||
# swarm's SSO — a boot-order fact rather than a preference. This store holds
|
||
# authelia's own OIDC client secret, so a client that had to obtain an authelia
|
||
# token first could never start from cold. `swarm-nats` can lean on authelia
|
||
# precisely because it does not store authelia's credentials.
|
||
#
|
||
# ⚠️ NO GATEWAY VHOST, and unlike `swarm-nats` that is not because this speaks
|
||
# a non-HTTP protocol. It speaks HTTPS, so nginx *could* front it: **the client
|
||
# certificate IS the authentication**, and a terminating proxy strips it,
|
||
# leaving bao seeing nginx as the client for every hive in the swarm — one
|
||
# identity where there must be many. Reach is loopback plus whatever
|
||
# `deploy.bao.extraListenAddresses` names.
|
||
#
|
||
# ⚠️ THIS MODULE HAS NO OPINION ABOUT WHERE THE STORE'S IDENTITY COMES FROM.
|
||
# A store must not take its certificates from an authority it will itself
|
||
# distribute: reach the store to get the CA material, need a cert from that CA
|
||
# to reach the store. Service↔store mTLS is therefore its own trust domain,
|
||
# separate from the gateway's HTTPS certificates and from both CAs in this
|
||
# tree. The cert paths are inputs this module declares no default for and never
|
||
# fills in; a glue module mints that identity and points them at it.
|
||
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
cfg = config.services.hyperhive.swarm.bao;
|
||
hyperhiveCfg = config.services.hyperhive;
|
||
deployCfg = hyperhiveCfg.deploy;
|
||
baoDeploy = deployCfg.bao;
|
||
networkCfg = hyperhiveCfg.network;
|
||
swarmDomain = hyperhiveCfg.swarm.domain;
|
||
|
||
# Upstream's own default, kept so its documentation matches. The raft data
|
||
# lives INSIDE the container on `ephemeral = false`, the same way
|
||
# ./swarm-grafana.nix keeps its sqlite database — no sibling service binds
|
||
# its state out to the host.
|
||
#
|
||
# ⚠️ Do not bind-mount this path. Upstream pairs `StateDirectory=` with
|
||
# `DynamicUser=`, which makes systemd hold the state at
|
||
# `/var/lib/private/openbao` and symlink this to it; that relocation is a
|
||
# rename, and a rename of an active mount point fails `EBUSY` at
|
||
# `STATE_DIRECTORY` — the unit then dies before `bao` runs at all.
|
||
stateDir = "/var/lib/openbao";
|
||
|
||
# The TLS material is its own small bind, on both sides of the boundary at
|
||
# the same path. Separate from `stateDir` because a HOST unit writes it and
|
||
# the container only reads it, and because systemd owns `stateDir` and moves
|
||
# it around; this directory is ours.
|
||
tlsDir = "/var/lib/swarm-bao-tls";
|
||
|
||
# Where the key surfaces for openbao to open. NOT `${tlsDir}/server-key.pem`:
|
||
# that file is root-owned 0600 on the host, and the service runs as a
|
||
# `DynamicUser`, so it cannot read the bind-mounted original. `LoadCredential`
|
||
# is systemd's answer to exactly this — PID 1 reads the source as root and
|
||
# re-exposes it inside the unit, owned by the service's own account.
|
||
# `$CREDENTIALS_DIRECTORY` is `/run/credentials/<unit>`, and the config file
|
||
# is rendered ahead of time, so the path is spelled out rather than read from
|
||
# the environment.
|
||
serverKeyCredential = "server-key";
|
||
serverKeyCredentialPath = "/run/credentials/openbao.service/${serverKeyCredential}";
|
||
|
||
# The PIN is deliberately absent here. It arrives as `BAO_HSM_PIN` from an
|
||
# EnvironmentFile the provisioning unit writes, because a value interpolated
|
||
# into a nix expression renders world-readable into the store.
|
||
#
|
||
# With no seal stanza openbao falls back to Shamir, so this attrset being
|
||
# empty is the difference between a store that unseals itself and one that
|
||
# needs a human after every restart.
|
||
# The PKCS11 token store and its PINs live on the HOST and are bind-mounted
|
||
# in. Losing them loses the sealed store — the raft data is worth nothing
|
||
# without the key that unseals it — so unlike the state directory these are
|
||
# deliberately a host-level fact an operator can back up.
|
||
tokenStoreDir = "/var/lib/swarm-bao-token";
|
||
pinEnvFile = "${tokenStoreDir}/pin.env";
|
||
|
||
sealSettings = lib.optionalAttrs (baoDeploy.seal == "pkcs11") {
|
||
seal.pkcs11 = {
|
||
lib = "${pkgs.tpm2-pkcs11}/lib/libtpm2_pkcs11.so";
|
||
token_label = "swarm-bao";
|
||
key_label = "swarm-bao-seal";
|
||
};
|
||
};
|
||
|
||
# Total on a null swarm domain for the same reason every sibling module is:
|
||
# the required-domain assertion in hive-network.nix should be what an operator
|
||
# sees, not a coercion error from here.
|
||
domainBase = if swarmDomain == null then "invalid" else swarmDomain;
|
||
|
||
# Where the leaf lands for openbao to read. `tlsDir` is bind-mounted at the
|
||
# same path on both sides, so the delivery below needs no second mount, and
|
||
# nothing has to bind `deploy.hive-controller.tls.stateDir`, which holds the
|
||
# hive CA's private key.
|
||
#
|
||
# The certificate and the client CA are public material and are read straight
|
||
# off the mount; only the key takes the credential path above.
|
||
serverCertPath = "${tlsDir}/server.pem";
|
||
serverKeyPath = "${tlsDir}/server-key.pem";
|
||
|
||
# The host-side sources, verbatim from the options — no fallback, because a
|
||
# fallback is exactly the CA opinion this module must not hold. The units
|
||
# below only exist when both are set (see `haveServerTls`), so these are
|
||
# never forced while null.
|
||
serverCertSrc = baoDeploy.serverCertFile;
|
||
serverKeySrc = baoDeploy.serverKeyFile;
|
||
|
||
# Both or neither: a certificate without its key configures a listener that
|
||
# cannot start, and the failure would surface as openbao refusing to boot
|
||
# rather than as the missing setting it is.
|
||
haveServerTls = baoDeploy.serverCertFile != null && baoDeploy.serverKeyFile != null;
|
||
|
||
# Every listener serves the same identity: they differ in which address
|
||
# they answer on, not in who they are. Client verification is separate and
|
||
# optional — a store with no `clientCaFile` still serves TLS, it just does
|
||
# not authenticate the far end, which is the honest rendering of "nobody
|
||
# has said what to trust yet".
|
||
listenerTls = {
|
||
tls_cert_file = serverCertPath;
|
||
tls_key_file = serverKeyCredentialPath;
|
||
}
|
||
// lib.optionalAttrs (baoDeploy.clientCaFile != null) {
|
||
tls_client_ca_file = clientCaPath;
|
||
tls_require_and_verify_client_cert = true;
|
||
};
|
||
|
||
clientCaPath = "${tlsDir}/client-ca.pem";
|
||
|
||
extraListeners = lib.listToAttrs (
|
||
lib.imap1 (
|
||
i: addr:
|
||
lib.nameValuePair "extra-${toString i}" (
|
||
{
|
||
type = "tcp";
|
||
address = "${addr}:${toString cfg.port}";
|
||
}
|
||
// listenerTls
|
||
)
|
||
) baoDeploy.extraListenAddresses
|
||
);
|
||
|
||
# Loopback is unconditional and everything else is declared, which is not
|
||
# symmetry for its own sake:
|
||
#
|
||
# A reader on this host reaches the store through loopback, and the host
|
||
# running the store is always one of its readers — so loopback is a property
|
||
# of what the store IS, not of where it sits. Every other address depends on
|
||
# which network the hives that read it share, and that is a deployment fact.
|
||
# Bind only loopback and no remote hive can reach the store; bind only a
|
||
# shared-network address and an all-local swarm cannot reach its own.
|
||
#
|
||
# Neither is a superset of the other, which is why this is not one address
|
||
# with a conditional value.
|
||
listeners = {
|
||
loopback = {
|
||
type = "tcp";
|
||
address = "127.0.0.1:${toString cfg.port}";
|
||
}
|
||
// listenerTls;
|
||
}
|
||
// extraListeners;
|
||
in
|
||
{
|
||
# One service, two namespaces, and the split decides who may set what.
|
||
#
|
||
# `deploy.bao.*` is what the host RUNNING the store decides: whether to run
|
||
# it (`enable`, declared in ./deploy.nix with its siblings), which build,
|
||
# how the root key is sealed, what it listens on. None of it means anything
|
||
# on a host that only reads secrets.
|
||
#
|
||
# `swarm.bao.*` below is what every host in the swarm has to agree on — the
|
||
# name the store answers to, its port, its container. A host that is purely
|
||
# a *client* needs all of that, because it is how the client finds the store.
|
||
options.services.hyperhive.deploy.bao = {
|
||
package = lib.mkOption {
|
||
type = lib.types.package;
|
||
default = pkgs.openbao;
|
||
defaultText = lib.literalExpression "pkgs.openbao";
|
||
description = ''
|
||
OpenBao package to run.
|
||
|
||
⚠️ An assertion below refuses 2.7.0 or newer, which drops the
|
||
built-in PKCS11 seal.
|
||
'';
|
||
};
|
||
|
||
seal = lib.mkOption {
|
||
type = lib.types.enum [
|
||
"pkcs11"
|
||
"shamir"
|
||
];
|
||
default = "pkcs11";
|
||
example = "shamir";
|
||
description = ''
|
||
How the store's root key is sealed.
|
||
|
||
`pkcs11` is the default and binds the key to the host's TPM: the store
|
||
unseals itself at boot, and an attacker with the disk does not get the
|
||
secrets. `shamir` is openbao's own default — unseal keys held by
|
||
whoever ran `bao operator init`, entered by hand after every restart —
|
||
and is the honest choice for a host with no TPM.
|
||
|
||
⚠️ This is a **declaration**, and nothing at evaluation time can check
|
||
it: nix runs on the build machine and cannot see the target's TPM.
|
||
Saying `pkcs11` on a host without one fails at activation, when the
|
||
provisioning unit cannot create the token. That is deliberate — a
|
||
store that comes up sealed by software while the config says hardware
|
||
is weaker than it reads, and silently so.
|
||
'';
|
||
};
|
||
|
||
serverCertFile = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/var/lib/swarm-bao/server.pem";
|
||
description = ''
|
||
Certificate the store serves, covering
|
||
{option}`services.hyperhive.swarm.bao.domain`.
|
||
|
||
This module declares no default and deliberately does not know
|
||
what could provide one — for the same reason
|
||
{option}`services.hyperhive.deploy.bao.clientCaFile` doesn't: the
|
||
store never reaches for an authority.
|
||
|
||
On a hive that runs the store, a glue module supplies a path as a
|
||
`mkDefault`, so naming your own here wins over it.
|
||
|
||
A path, never a value.
|
||
'';
|
||
};
|
||
|
||
serverKeyFile = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/var/lib/swarm-bao/server-key.pem";
|
||
description = ''
|
||
Private key for {option}`services.hyperhive.deploy.bao.serverCertFile`.
|
||
Both or neither — a certificate with no key is a listener that cannot
|
||
start.
|
||
'';
|
||
};
|
||
|
||
clientCaFile = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/var/lib/swarm-ca/root.pem";
|
||
description = ''
|
||
Authority the store validates hive **client** certificates against.
|
||
|
||
This module declares no default and does not reach for the hive
|
||
CA: the hive CA is a future *consumer* of the store, so a store
|
||
that authenticated against it could not come up before the thing
|
||
it issues.
|
||
|
||
On a hive that runs the store, a glue module supplies the CA it
|
||
minted for exactly this, as a `mkDefault`. Point this at something
|
||
else — the swarm root, an operator's own CA — and yours wins.
|
||
|
||
`null` leaves client-certificate verification off, which is only
|
||
appropriate where something else authenticates the connection.
|
||
'';
|
||
};
|
||
|
||
extraListenAddresses = lib.mkOption {
|
||
type = lib.types.listOf lib.types.str;
|
||
default = [ ];
|
||
example = [ "10.100.0.1" ];
|
||
description = ''
|
||
Addresses the store listens on **in addition to loopback**, each on
|
||
{option}`services.hyperhive.swarm.bao.port`.
|
||
|
||
Loopback is unconditional and not listed here: the host running the
|
||
store is always one of its readers. Every other address depends on
|
||
which network the reading hives share with this one, and that is a
|
||
deployment fact no other module's config can be read to infer — a
|
||
swarm meshed over wireguard names its mesh address, one on a trusted
|
||
LAN names that interface, and an all-local swarm names nothing at all.
|
||
|
||
Addresses only, no port: a store reachable on two ports is a
|
||
misconfiguration rather than a topology.
|
||
'';
|
||
};
|
||
};
|
||
|
||
options.services.hyperhive.swarm.bao = {
|
||
machine = lib.mkOption {
|
||
type = lib.types.str;
|
||
readOnly = true;
|
||
default = "swarm-bao";
|
||
description = ''
|
||
Container name. Read-only: the name appears in host paths and in
|
||
`machinectl`, so it is a fact other modules may read rather than a knob.
|
||
'';
|
||
};
|
||
|
||
domain = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "bao.${domainBase}";
|
||
defaultText = lib.literalExpression ''"bao.''${services.hyperhive.swarm.domain}"'';
|
||
description = ''
|
||
Name the store is reached on. A **sibling** of the swarm's other
|
||
service names, not a child of any hive domain: an authority whose
|
||
`nameConstraints` permit one hive's domain cannot issue for a sibling
|
||
of it, so the shape of this name decides which authorities could ever
|
||
sign for the store. That is a property of the name, not a choice of
|
||
issuer — this module makes no such choice.
|
||
'';
|
||
};
|
||
|
||
port = lib.mkOption {
|
||
type = lib.types.port;
|
||
default = 8200;
|
||
description = ''
|
||
TCP port the store listens on. Upstream's own default, kept so an
|
||
operator reading OpenBao documentation finds what they expect.
|
||
|
||
Swarm-wide because a client has to know it to reach the store, and
|
||
the same port on every listener: which *addresses* the store answers
|
||
on is the running host's business
|
||
({option}`services.hyperhive.deploy.bao.extraListenAddresses`), but
|
||
which port it answers on is something the whole swarm agrees.
|
||
'';
|
||
};
|
||
|
||
};
|
||
|
||
# ⚠️ Gated on `deploy.bao.enable`, and that is load-bearing rather than
|
||
# tidiness: an unconditional `config` block would evaluate the seal
|
||
# assertion on EVERY hive, so a hive that runs no secret store at all
|
||
# would fail to build the day nixpkgs moves openbao past 2.7.0. A check
|
||
# about running this service has no business firing where it is not run.
|
||
config = lib.mkMerge [
|
||
# Assertions sit in their own arm, gated only on running the store, so
|
||
# they still fire when the cert paths are unset — the arm below is not
|
||
# evaluated in that case, and an assertion that disappears exactly when
|
||
# its subject is broken would be worse than none.
|
||
(lib.mkIf (hyperhiveCfg.enable && deployCfg.bao.enable) {
|
||
assertions = [
|
||
{
|
||
assertion = lib.versionOlder baoDeploy.package.version "2.7.0";
|
||
message = ''
|
||
The swarm secret store needs openbao older than 2.7.0 (this is
|
||
${baoDeploy.package.version}). 2.7.0 moves the PKCS11 seal out of the
|
||
distribution into a plugin nixpkgs does not package, so the store
|
||
would come up sealed by software without saying so.
|
||
See https://openbao.org/community/deprecation/
|
||
'';
|
||
}
|
||
{
|
||
assertion = haveServerTls;
|
||
message = ''
|
||
The swarm secret store has no server certificate: set both
|
||
services.hyperhive.deploy.bao.serverCertFile and .serverKeyFile.
|
||
|
||
This module defaults neither, on purpose — a store must not
|
||
take its identity from an authority it will itself distribute,
|
||
and service-to-store mTLS is a separate trust domain from the
|
||
gateway's certificates and from either CA in this tree.
|
||
|
||
A hive that runs the store normally gets both from a glue
|
||
module, so reaching this means that glue is absent or
|
||
something set these back to null.
|
||
'';
|
||
}
|
||
];
|
||
})
|
||
|
||
(lib.mkIf (hyperhiveCfg.enable && deployCfg.bao.enable && haveServerTls) {
|
||
# Provisions the TPM-backed token the seal above names. One-shot and
|
||
# idempotent on ABSENCE, never on content: regenerating a PIN would
|
||
# orphan an already-sealed store, so a rebuild must not rotate it.
|
||
#
|
||
# ⚠️ Untestable without a TPM, and the failure is deliberately at
|
||
# activation — nix evaluates on the build machine and cannot see the
|
||
# target's hardware, so `seal = "pkcs11"` is a declaration this unit
|
||
# either makes true or fails on.
|
||
# The store's server certificate, delivered rather than bind-mounted.
|
||
#
|
||
# ⚠️ A copy, for three separate reasons — the last one is the one that
|
||
# matters most and is the least obvious:
|
||
# 1. `nixos-container` refuses to start when a bind source is missing,
|
||
# and a certificate minted on this same boot does not exist yet when
|
||
# the container is ordered. Same trap `hostClientSecretDir`
|
||
# documents in ./swarm-authelia.nix. Ordering against whatever
|
||
# mints it belongs with whatever named the path, not here.
|
||
# 2. `tlsDir` is already mounted at the same path inside, so a copy
|
||
# needs no second mount.
|
||
# 3. A directory holding a leaf usually holds the CA's private key
|
||
# beside it. Binding that directory to reach one file inside it
|
||
# would hand the container authority to mint any name that CA can —
|
||
# which is why this takes a path to a FILE and copies it.
|
||
systemd.services.swarm-bao-certs = {
|
||
description = "deliver the swarm secret store's server certificate";
|
||
before = [ "container@${cfg.machine}.service" ];
|
||
requiredBy = [ "container@${cfg.machine}.service" ];
|
||
path = [ pkgs.coreutils ];
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
};
|
||
script = ''
|
||
set -euo pipefail
|
||
# 0755, not 0700: the container reads the certificate and the client
|
||
# CA straight off this mount as a non-root user, so it has to be able
|
||
# to traverse the directory. The key inside stays 0600 and reaches
|
||
# the service through `LoadCredential` instead.
|
||
install -d -m 0755 ${tlsDir}
|
||
|
||
# Fail loudly rather than start a store that cannot serve. The path
|
||
# is configured, so a missing file means whatever was supposed to
|
||
# produce it did not run or failed — either way this is where it is
|
||
# cheapest to notice. Otherwise it surfaces at the TLS handshake,
|
||
# several layers from the setting that caused it.
|
||
for f in ${lib.escapeShellArg serverCertSrc} ${lib.escapeShellArg serverKeySrc}; do
|
||
if [ ! -s "$f" ]; then
|
||
echo "swarm-bao has no server certificate: $f is missing or empty." >&2
|
||
echo "That path comes from deploy.bao.serverCertFile/serverKeyFile." >&2
|
||
exit 1
|
||
fi
|
||
done
|
||
|
||
install -m 0644 ${lib.escapeShellArg serverCertSrc} ${tlsDir}/server.pem
|
||
install -m 0600 ${lib.escapeShellArg serverKeySrc} ${tlsDir}/server-key.pem
|
||
''
|
||
+ lib.optionalString (baoDeploy.clientCaFile != null) ''
|
||
|
||
if [ ! -s ${lib.escapeShellArg baoDeploy.clientCaFile} ]; then
|
||
echo "deploy.bao.clientCaFile names ${baoDeploy.clientCaFile}, which is missing or empty." >&2
|
||
exit 1
|
||
fi
|
||
install -m 0644 ${lib.escapeShellArg baoDeploy.clientCaFile} ${tlsDir}/client-ca.pem
|
||
'';
|
||
};
|
||
|
||
systemd.services.swarm-bao-token = lib.mkIf (baoDeploy.seal == "pkcs11") {
|
||
description = "provision the swarm secret store's TPM-backed PKCS11 token";
|
||
before = [ "container@${cfg.machine}.service" ];
|
||
requiredBy = [ "container@${cfg.machine}.service" ];
|
||
path = [
|
||
pkgs.openssl
|
||
pkgs.tpm2-pkcs11
|
||
pkgs.tpm2-tools
|
||
];
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
};
|
||
script = ''
|
||
set -euo pipefail
|
||
install -d -m 0700 ${tokenStoreDir}
|
||
# Required whenever the store is not at its default location, or the
|
||
# library cannot find the token the seal asks for.
|
||
export TPM2_PKCS11_STORE=${tokenStoreDir}
|
||
|
||
# Absence is the only trigger. `openssl rand` is the same generator
|
||
# the grafana admin key uses; the value never passes through a nix
|
||
# expression, which would render it world-readable into the store.
|
||
for p in so-pin user-pin; do
|
||
if [ ! -e ${tokenStoreDir}/$p ]; then
|
||
( umask 077; openssl rand -hex 16 > ${tokenStoreDir}/$p )
|
||
chmod 0400 ${tokenStoreDir}/$p
|
||
fi
|
||
done
|
||
|
||
if [ ! -e ${tokenStoreDir}/tpm2_pkcs11.sqlite3 ]; then
|
||
pid=$(tpm2_ptool init --path ${tokenStoreDir} | sed -n 's/.*id: //p')
|
||
tpm2_ptool addtoken --path ${tokenStoreDir} --pid="$pid" \
|
||
--label=swarm-bao \
|
||
--sopin="$(cat ${tokenStoreDir}/so-pin)" \
|
||
--userpin="$(cat ${tokenStoreDir}/user-pin)"
|
||
# AES rather than RSA on purpose: openbao discussion 1826
|
||
# reports an RSA keypair here yielding duplicate labels, so
|
||
# `bao operator init` fails with "got more than 1 key for the
|
||
# label" and then CKR_GENERAL_ERROR. The seal supports AES-GCM.
|
||
tpm2_ptool addkey --path ${tokenStoreDir} --label=swarm-bao \
|
||
--userpin="$(cat ${tokenStoreDir}/user-pin)" \
|
||
--algorithm=aes256 --key-label=swarm-bao-seal
|
||
fi
|
||
|
||
( umask 077; printf 'BAO_HSM_PIN=%s\n' "$(cat ${tokenStoreDir}/user-pin)" > ${pinEnvFile} )
|
||
chmod 0400 ${pinEnvFile}
|
||
'';
|
||
};
|
||
|
||
containers.${cfg.machine} = {
|
||
autoStart = true;
|
||
ephemeral = false;
|
||
# Shared host netns, like every sibling swarm container. Unlike them the
|
||
# gateway is NOT the client here (see the no-vhost note at the top), so
|
||
# sharing the netns is what lets the store bind the host's own addresses
|
||
# rather than a convenience for nginx.
|
||
privateNetwork = false;
|
||
|
||
# Only what a HOST unit writes and this container reads crosses the
|
||
# boundary. The raft state deliberately does not: `ephemeral = false`
|
||
# keeps the container's own /var, systemd owns `${stateDir}` through
|
||
# `StateDirectory=`, and binding over it is what breaks the unit.
|
||
bindMounts = {
|
||
# Read-only: `swarm-bao-certs` on the host is the only writer, and
|
||
# the store has no reason to modify its own identity.
|
||
${tlsDir} = {
|
||
hostPath = tlsDir;
|
||
isReadOnly = true;
|
||
};
|
||
}
|
||
// lib.optionalAttrs (baoDeploy.seal == "pkcs11") {
|
||
# Writable: the library keeps its sqlite store here, and the seal
|
||
# reads the token through it on every unseal.
|
||
${tokenStoreDir} = {
|
||
hostPath = tokenStoreDir;
|
||
isReadOnly = false;
|
||
};
|
||
};
|
||
|
||
# The seal talks to the TPM through the kernel's resource manager, so
|
||
# the device has to cross the container boundary or the store cannot
|
||
# unseal itself — which is the whole point of pkcs11 over shamir.
|
||
allowedDevices = lib.optionals (baoDeploy.seal == "pkcs11") [
|
||
{
|
||
node = "/dev/tpmrm0";
|
||
modifier = "rw";
|
||
}
|
||
];
|
||
|
||
config =
|
||
{ ... }:
|
||
{
|
||
imports = [
|
||
(import ./swarm-container-resolver.nix {
|
||
inherit (networkCfg) bridgeIp;
|
||
dnsConsumers = [ "openbao.service" ];
|
||
})
|
||
];
|
||
|
||
system.stateVersion = "26.05";
|
||
|
||
# Shares the host netns, so its own firewall.service would rewrite
|
||
# the HOST ruleset at every boot. The host firewall owns filtering.
|
||
networking.firewall.enable = false;
|
||
# The resolver unit imported above owns /etc/resolv.conf; leaving
|
||
# resolvconf on would let host-tracking regenerate it empty.
|
||
networking.resolvconf.enable = lib.mkForce false;
|
||
|
||
services.openbao = {
|
||
enable = true;
|
||
package = baoDeploy.package;
|
||
settings = {
|
||
listener = listeners;
|
||
storage.raft.path = stateDir;
|
||
}
|
||
// sealSettings;
|
||
};
|
||
|
||
# The private key crosses the user boundary here, not on the mount.
|
||
# `swarm-bao-certs` installs it 0600 root-owned, and the unit runs
|
||
# as a `DynamicUser`, so the bind-mounted file is unreadable to it —
|
||
# PID 1 opens the source as root and re-exposes it under
|
||
# `${serverKeyCredentialPath}`, owned by the service's own account.
|
||
#
|
||
# One assignment, not two: `serviceConfig.X = …` beside a
|
||
# `serviceConfig = …` is a duplicate attribute inside a single
|
||
# attrset literal and does not parse. Module merging happens across
|
||
# `config` blocks, not within a literal — so the seal's half joins
|
||
# with `optionalAttrs`.
|
||
systemd.services.openbao.serviceConfig =
|
||
lib.optionalAttrs haveServerTls {
|
||
LoadCredential = [ "${serverKeyCredential}:${serverKeyPath}" ];
|
||
}
|
||
# The PIN reaches openbao as an environment variable read from a
|
||
# 0400 file the provisioning unit wrote — never as a value in this
|
||
# expression, which would render it world-readable into the store.
|
||
# `TPM2_PKCS11_STORE` is required because the store is not at the
|
||
# library's default location.
|
||
// lib.optionalAttrs (baoDeploy.seal == "pkcs11") {
|
||
EnvironmentFile = pinEnvFile;
|
||
Environment = [ "TPM2_PKCS11_STORE=${tokenStoreDir}" ];
|
||
};
|
||
|
||
# ⚠️ Upstream sets `restartIfChanged = false` on this unit, on
|
||
# purpose: a restart SEALS the store and disconnects every client.
|
||
# So a change to the settings above does NOT take effect on
|
||
# `nixos-rebuild switch` — it lands in the config file and waits.
|
||
# Restarting is an operator action with an unseal on the far side of
|
||
# it, which is why nothing here tries to be clever about it.
|
||
};
|
||
};
|
||
})
|
||
|
||
];
|
||
}
|