diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index 8c180f4f..7683d730 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -24,6 +24,7 @@ ./hive-tls.nix ./otel.nix ./swarm-authelia.nix + ./swarm-bao.nix ./swarm-ca.nix ./swarm-nats.nix ./swarm-controller.nix diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix new file mode 100644 index 00000000..37802f39 --- /dev/null +++ b/nix/host-modules/swarm-bao.nix @@ -0,0 +1,542 @@ +# The swarm's secret store: one OpenBao for the whole swarm, in a +# `swarm-bao` nixos-container. +# +# Today every credential in ./swarm-*.nix is minted where it is read or copied +# there by a delivery unit (docs/swarm/secrets.md), which ties each secret's +# lifetime to its container's. +# +# ⚠️ It authenticates hive clients with a CLIENT CERTIFICATE, not with the +# swarm's SSO — a boot-order fact rather than a preference. This store holds +# authelia's own OIDC client secret, so a client that had to obtain an authelia +# token first could never start from cold. `swarm-nats` can lean on authelia +# precisely because it does not store authelia's credentials. +# +# ⚠️ NO GATEWAY VHOST, and unlike `swarm-nats` that is not because this speaks +# a non-HTTP protocol. It speaks HTTPS, so nginx *could* front it: **the client +# certificate IS the authentication**, and a terminating proxy strips it, +# leaving bao seeing nginx as the client for every hive in the swarm — one +# identity where there must be many. Reach is loopback plus whatever +# `deploy.bao.extraListenAddresses` names. +# +# ⚠️ THIS MODULE HAS NO OPINION ABOUT WHERE THE STORE'S IDENTITY COMES FROM. +# A store must not take its certificates from an authority it will itself +# distribute: reach the store to get the CA material, need a cert from that CA +# to reach the store. Service↔store mTLS is therefore its own trust domain, +# separate from the gateway's HTTPS certificates and from both CAs in this +# tree. The cert paths are inputs with no defaults and nothing here fills them +# in; whatever comes to mint that identity is what they will point at. +{ + pkgs, + lib, + config, + ... +}: +let + cfg = config.services.hyperhive.swarm.bao; + hyperhiveCfg = config.services.hyperhive; + deployCfg = hyperhiveCfg.deploy; + baoDeploy = deployCfg.bao; + networkCfg = hyperhiveCfg.network; + swarmDomain = hyperhiveCfg.swarm.domain; + + # Two names for one location. `stateDir` is where openbao writes inside the + # container — upstream's own default, kept so its documentation matches. The + # host path is a sibling of the other swarm services' state rather than a + # path inside the container's tree, so `nixos-container destroy` cannot take + # the swarm's secrets with it. + stateDir = "/var/lib/openbao"; + hostStateDir = "/var/lib/swarm-bao"; + + # The PIN is deliberately absent here. It arrives as `BAO_HSM_PIN` from an + # EnvironmentFile the provisioning unit writes, because a value interpolated + # into a nix expression renders world-readable into the store. + # + # With no seal stanza openbao falls back to Shamir, so this attrset being + # empty is the difference between a store that unseals itself and one that + # needs a human after every restart. + # The PKCS11 token store and its PINs live on the HOST and are bind-mounted + # in. Losing them loses the sealed store, so they are a host-level fact an + # operator can back up — the same reasoning as the raft data below. + tokenStoreDir = "/var/lib/swarm-bao-token"; + pinEnvFile = "${tokenStoreDir}/pin.env"; + + sealSettings = lib.optionalAttrs (baoDeploy.seal == "pkcs11") { + seal.pkcs11 = { + lib = "${pkgs.tpm2-pkcs11}/lib/libtpm2_pkcs11.so"; + token_label = "swarm-bao"; + key_label = "swarm-bao-seal"; + }; + }; + + # Total on a null swarm domain for the same reason every sibling module is: + # the required-domain assertion in hive-network.nix should be what an operator + # sees, not a coercion error from here. + domainBase = if swarmDomain == null then "invalid" else swarmDomain; + + # Where the leaf lands for openbao to read. Inside the container, because + # `hostStateDir` is already bind-mounted at `stateDir` — so the delivery + # below needs no second mount, and nothing has to bind `tls.stateDir`, + # which holds the hive CA's private key. + serverCertPath = "${stateDir}/server.pem"; + serverKeyPath = "${stateDir}/server-key.pem"; + + # The host-side sources, verbatim from the options — no fallback, because a + # fallback is exactly the CA opinion this module must not hold. The units + # below only exist when both are set (see `haveServerTls`), so these are + # never forced while null. + serverCertSrc = baoDeploy.serverCertFile; + serverKeySrc = baoDeploy.serverKeyFile; + + # Both or neither: a certificate without its key configures a listener that + # cannot start, and the failure would surface as openbao refusing to boot + # rather than as the missing setting it is. + haveServerTls = baoDeploy.serverCertFile != null && baoDeploy.serverKeyFile != null; + + # Every listener serves the same identity: they differ in which address + # they answer on, not in who they are. Client verification is separate and + # optional — a store with no `clientCaFile` still serves TLS, it just does + # not authenticate the far end, which is the honest rendering of "nobody + # has said what to trust yet". + listenerTls = { + tls_cert_file = serverCertPath; + tls_key_file = serverKeyPath; + } + // lib.optionalAttrs (baoDeploy.clientCaFile != null) { + tls_client_ca_file = clientCaPath; + tls_require_and_verify_client_cert = true; + }; + + clientCaPath = "${stateDir}/client-ca.pem"; + + extraListeners = lib.listToAttrs ( + lib.imap1 ( + i: addr: + lib.nameValuePair "extra-${toString i}" ( + { + type = "tcp"; + address = "${addr}:${toString cfg.port}"; + } + // listenerTls + ) + ) baoDeploy.extraListenAddresses + ); + + # Loopback is unconditional and everything else is declared, which is not + # symmetry for its own sake: + # + # A reader on this host reaches the store through loopback, and the host + # running the store is always one of its readers — so loopback is a property + # of what the store IS, not of where it sits. Every other address depends on + # which network the hives that read it share, and that is a deployment fact. + # Bind only loopback and no remote hive can reach the store; bind only a + # shared-network address and an all-local swarm cannot reach its own. + # + # Neither is a superset of the other, which is why this is not one address + # with a conditional value. + listeners = { + loopback = { + type = "tcp"; + address = "127.0.0.1:${toString cfg.port}"; + } + // listenerTls; + } + // extraListeners; +in +{ + # One service, two namespaces, and the split decides who may set what. + # + # `deploy.bao.*` is what the host RUNNING the store decides: whether to run + # it (`enable`, declared in ./deploy.nix with its siblings), which build, + # how the root key is sealed, what it listens on. None of it means anything + # on a host that only reads secrets. + # + # `swarm.bao.*` below is what every host in the swarm has to agree on — the + # name the store answers to, its port, its container. A host that is purely + # a *client* needs all of that, because it is how the client finds the store. + options.services.hyperhive.deploy.bao = { + package = lib.mkOption { + type = lib.types.package; + default = pkgs.openbao; + defaultText = lib.literalExpression "pkgs.openbao"; + description = '' + OpenBao package to run. + + ⚠️ An assertion below refuses 2.7.0 or newer, which drops the + built-in PKCS11 seal. + ''; + }; + + seal = lib.mkOption { + type = lib.types.enum [ + "pkcs11" + "shamir" + ]; + default = "pkcs11"; + example = "shamir"; + description = '' + How the store's root key is sealed. + + `pkcs11` is the default and binds the key to the host's TPM: the store + unseals itself at boot, and an attacker with the disk does not get the + secrets. `shamir` is openbao's own default — unseal keys held by + whoever ran `bao operator init`, entered by hand after every restart — + and is the honest choice for a host with no TPM. + + ⚠️ This is a **declaration**, and nothing at evaluation time can check + it: nix runs on the build machine and cannot see the target's TPM. + Saying `pkcs11` on a host without one fails at activation, when the + provisioning unit cannot create the token. That is deliberate — a + store that comes up sealed by software while the config says hardware + is weaker than it reads, and silently so. + ''; + }; + + serverCertFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-bao/server.pem"; + description = '' + Certificate the store serves, covering + {option}`services.hyperhive.swarm.bao.domain`. + + No default, and this module deliberately does not know what could + provide one — for the same reason + {option}`services.hyperhive.deploy.bao.clientCaFile` doesn't. The + deployment names the file; the store never reaches for an authority. + + A path, never a value. + ''; + }; + + serverKeyFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-bao/server-key.pem"; + description = '' + Private key for {option}`services.hyperhive.deploy.bao.serverCertFile`. + Both or neither — a certificate with no key is a listener that cannot + start. + ''; + }; + + clientCaFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-ca/root.pem"; + description = '' + Authority the store validates hive **client** certificates against. + + Deliberately has no default, and does not reach for the hive CA: + the hive CA is a future *consumer* of the store, so a store that + authenticated against it could not come up before the thing it + issues. It is a value someone points at — the swarm root for a + swarm that runs one, an operator's own CA otherwise. + + `null` leaves client-certificate verification off, which is only + appropriate where something else authenticates the connection. + ''; + }; + + extraListenAddresses = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + example = [ "10.100.0.1" ]; + description = '' + Addresses the store listens on **in addition to loopback**, each on + {option}`services.hyperhive.swarm.bao.port`. + + Loopback is unconditional and not listed here: the host running the + store is always one of its readers. Every other address depends on + which network the reading hives share with this one, and that is a + deployment fact no other module's config can be read to infer — a + swarm meshed over wireguard names its mesh address, one on a trusted + LAN names that interface, and an all-local swarm names nothing at all. + + Addresses only, no port: a store reachable on two ports is a + misconfiguration rather than a topology. + ''; + }; + }; + + options.services.hyperhive.swarm.bao = { + machine = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = "swarm-bao"; + description = '' + Container name. Read-only: the name appears in host paths and in + `machinectl`, so it is a fact other modules may read rather than a knob. + ''; + }; + + domain = lib.mkOption { + type = lib.types.str; + default = "bao.${domainBase}"; + defaultText = lib.literalExpression ''"bao.''${services.hyperhive.swarm.domain}"''; + description = '' + Name the store is reached on. A **sibling** of the swarm's other + service names, not a child of any hive domain: an authority whose + `nameConstraints` permit one hive's domain cannot issue for a sibling + of it, so the shape of this name decides which authorities could ever + sign for the store. That is a property of the name, not a choice of + issuer — this module makes no such choice. + ''; + }; + + port = lib.mkOption { + type = lib.types.port; + default = 8200; + description = '' + TCP port the store listens on. Upstream's own default, kept so an + operator reading OpenBao documentation finds what they expect. + + Swarm-wide because a client has to know it to reach the store, and + the same port on every listener: which *addresses* the store answers + on is the running host's business + ({option}`services.hyperhive.deploy.bao.extraListenAddresses`), but + which port it answers on is something the whole swarm agrees. + ''; + }; + + }; + + # ⚠️ Gated on `deploy.bao.enable`, and that is load-bearing rather than + # tidiness: an unconditional `config` block would evaluate the seal + # assertion on EVERY hive, so a hive that runs no secret store at all + # would fail to build the day nixpkgs moves openbao past 2.7.0. A check + # about running this service has no business firing where it is not run. + config = lib.mkMerge [ + # Assertions sit in their own arm, gated only on running the store, so + # they still fire when the cert paths are unset — the arm below is not + # evaluated in that case, and an assertion that disappears exactly when + # its subject is broken would be worse than none. + (lib.mkIf (hyperhiveCfg.enable && deployCfg.bao.enable) { + assertions = [ + { + assertion = lib.versionOlder baoDeploy.package.version "2.7.0"; + message = '' + The swarm secret store needs openbao older than 2.7.0 (this is + ${baoDeploy.package.version}). 2.7.0 moves the PKCS11 seal out of the + distribution into a plugin nixpkgs does not package, so the store + would come up sealed by software without saying so. + See https://openbao.org/community/deprecation/ + ''; + } + { + assertion = haveServerTls; + message = '' + The swarm secret store has no server certificate: set both + services.hyperhive.deploy.bao.serverCertFile and .serverKeyFile. + + Nothing defaults them, on purpose — a store must not take its + identity from an authority it will itself distribute, and + service-to-store mTLS is a separate trust domain from the + gateway's certificates and from either CA in this tree. + ''; + } + ]; + }) + + (lib.mkIf (hyperhiveCfg.enable && deployCfg.bao.enable && haveServerTls) { + # Provisions the TPM-backed token the seal above names. One-shot and + # idempotent on ABSENCE, never on content: regenerating a PIN would + # orphan an already-sealed store, so a rebuild must not rotate it. + # + # ⚠️ Untestable without a TPM, and the failure is deliberately at + # activation — nix evaluates on the build machine and cannot see the + # target's hardware, so `seal = "pkcs11"` is a declaration this unit + # either makes true or fails on. + # The store's server certificate, delivered rather than bind-mounted. + # + # ⚠️ A copy, for three separate reasons — the last one is the one that + # matters most and is the least obvious: + # 1. `nixos-container` refuses to start when a bind source is missing, + # and a certificate minted on this same boot does not exist yet when + # the container is ordered. Same trap `hostClientSecretDir` + # documents in ./swarm-authelia.nix. Ordering against whatever + # mints it belongs with whatever named the path, not here. + # 2. `hostStateDir` is already mounted at `stateDir`, so a copy needs + # no second mount. + # 3. A directory holding a leaf usually holds the CA's private key + # beside it. Binding that directory to reach one file inside it + # would hand the container authority to mint any name that CA can — + # which is why this takes a path to a FILE and copies it. + systemd.services.swarm-bao-certs = { + description = "deliver the swarm secret store's server certificate"; + before = [ "container@${cfg.machine}.service" ]; + requiredBy = [ "container@${cfg.machine}.service" ]; + path = [ pkgs.coreutils ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + script = '' + set -euo pipefail + install -d -m 0700 ${hostStateDir} + + # Fail loudly rather than start a store that cannot serve. The path + # is configured, so a missing file means whatever was supposed to + # produce it did not run or failed — either way this is where it is + # cheapest to notice. Otherwise it surfaces at the TLS handshake, + # several layers from the setting that caused it. + for f in ${lib.escapeShellArg serverCertSrc} ${lib.escapeShellArg serverKeySrc}; do + if [ ! -s "$f" ]; then + echo "swarm-bao has no server certificate: $f is missing or empty." >&2 + echo "That path comes from deploy.bao.serverCertFile/serverKeyFile." >&2 + exit 1 + fi + done + + install -m 0644 ${lib.escapeShellArg serverCertSrc} ${hostStateDir}/server.pem + install -m 0600 ${lib.escapeShellArg serverKeySrc} ${hostStateDir}/server-key.pem + '' + + lib.optionalString (baoDeploy.clientCaFile != null) '' + + if [ ! -s ${lib.escapeShellArg baoDeploy.clientCaFile} ]; then + echo "deploy.bao.clientCaFile names ${baoDeploy.clientCaFile}, which is missing or empty." >&2 + exit 1 + fi + install -m 0644 ${lib.escapeShellArg baoDeploy.clientCaFile} ${hostStateDir}/client-ca.pem + ''; + }; + + systemd.services.swarm-bao-token = lib.mkIf (baoDeploy.seal == "pkcs11") { + description = "provision the swarm secret store's TPM-backed PKCS11 token"; + before = [ "container@${cfg.machine}.service" ]; + requiredBy = [ "container@${cfg.machine}.service" ]; + path = [ + pkgs.openssl + pkgs.tpm2-pkcs11 + pkgs.tpm2-tools + ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + script = '' + set -euo pipefail + install -d -m 0700 ${tokenStoreDir} + # Required whenever the store is not at its default location, or the + # library cannot find the token the seal asks for. + export TPM2_PKCS11_STORE=${tokenStoreDir} + + # Absence is the only trigger. `openssl rand` is the same generator + # the grafana admin key uses; the value never passes through a nix + # expression, which would render it world-readable into the store. + for p in so-pin user-pin; do + if [ ! -e ${tokenStoreDir}/$p ]; then + ( umask 077; openssl rand -hex 16 > ${tokenStoreDir}/$p ) + chmod 0400 ${tokenStoreDir}/$p + fi + done + + if [ ! -e ${tokenStoreDir}/tpm2_pkcs11.sqlite3 ]; then + pid=$(tpm2_ptool init --path ${tokenStoreDir} | sed -n 's/.*id: //p') + tpm2_ptool addtoken --path ${tokenStoreDir} --pid="$pid" \ + --label=swarm-bao \ + --sopin="$(cat ${tokenStoreDir}/so-pin)" \ + --userpin="$(cat ${tokenStoreDir}/user-pin)" + # AES rather than RSA on purpose: openbao discussion 1826 + # reports an RSA keypair here yielding duplicate labels, so + # `bao operator init` fails with "got more than 1 key for the + # label" and then CKR_GENERAL_ERROR. The seal supports AES-GCM. + tpm2_ptool addkey --path ${tokenStoreDir} --label=swarm-bao \ + --userpin="$(cat ${tokenStoreDir}/user-pin)" \ + --algorithm=aes256 --key-label=swarm-bao-seal + fi + + ( umask 077; printf 'BAO_HSM_PIN=%s\n' "$(cat ${tokenStoreDir}/user-pin)" > ${pinEnvFile} ) + chmod 0400 ${pinEnvFile} + ''; + }; + + containers.${cfg.machine} = { + autoStart = true; + ephemeral = false; + # Shared host netns, like every sibling swarm container. Unlike them the + # gateway is NOT the client here (see the no-vhost note at the top), so + # sharing the netns is what lets the store bind the host's own addresses + # rather than a convenience for nginx. + privateNetwork = false; + + # Raft state outlives the container. `ephemeral = false` keeps the + # container's own /var, but a bind makes the store's data a host-level + # fact an operator can back up and a `nixos-container destroy` cannot + # take with it — which for the swarm's secrets is the difference between + # a rebuild and an outage. + bindMounts = { + ${stateDir} = { + hostPath = hostStateDir; + isReadOnly = false; + }; + } + // lib.optionalAttrs (baoDeploy.seal == "pkcs11") { + # Writable: the library keeps its sqlite store here, and the seal + # reads the token through it on every unseal. + ${tokenStoreDir} = { + hostPath = tokenStoreDir; + isReadOnly = false; + }; + }; + + # The seal talks to the TPM through the kernel's resource manager, so + # the device has to cross the container boundary or the store cannot + # unseal itself — which is the whole point of pkcs11 over shamir. + allowedDevices = lib.optionals (baoDeploy.seal == "pkcs11") [ + { + node = "/dev/tpmrm0"; + modifier = "rw"; + } + ]; + + config = + { ... }: + { + imports = [ + (import ./swarm-container-resolver.nix { + inherit (networkCfg) bridgeIp; + dnsConsumers = [ "openbao.service" ]; + }) + ]; + + system.stateVersion = "26.05"; + + # Shares the host netns, so its own firewall.service would rewrite + # the HOST ruleset at every boot. The host firewall owns filtering. + networking.firewall.enable = false; + # The resolver unit imported above owns /etc/resolv.conf; leaving + # resolvconf on would let host-tracking regenerate it empty. + networking.resolvconf.enable = lib.mkForce false; + + services.openbao = { + enable = true; + package = baoDeploy.package; + settings = { + listener = listeners; + storage.raft.path = stateDir; + } + // sealSettings; + }; + + # The PIN reaches openbao as an environment variable read from a + # 0400 file the provisioning unit wrote — never as a value in this + # expression, which would render it world-readable into the store. + # `TPM2_PKCS11_STORE` is required because the store is not at the + # library's default location. + systemd.services.openbao.serviceConfig = lib.mkIf (baoDeploy.seal == "pkcs11") { + EnvironmentFile = pinEnvFile; + Environment = [ "TPM2_PKCS11_STORE=${tokenStoreDir}" ]; + }; + + # ⚠️ Upstream sets `restartIfChanged = false` on this unit, on + # purpose: a restart SEALS the store and disconnects every client. + # So a change to the settings above does NOT take effect on + # `nixos-rebuild switch` — it lands in the config file and waits. + # Restarting is an operator action with an unseal on the far side of + # it, which is why nothing here tries to be clever about it. + }; + }; + }) + + ]; +}