diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index cdad7b38..501569f8 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -39,13 +39,34 @@ let networkCfg = hyperhiveCfg.network; swarmDomain = hyperhiveCfg.swarm.domain; - # Two names for one location. `stateDir` is where openbao writes inside the - # container — upstream's own default, kept so its documentation matches. The - # host path is a sibling of the other swarm services' state rather than a - # path inside the container's tree, so `nixos-container destroy` cannot take - # the swarm's secrets with it. + # Upstream's own default, kept so its documentation matches. The raft data + # lives INSIDE the container on `ephemeral = false`, the same way + # ./swarm-grafana.nix keeps its sqlite database — no sibling service binds + # its state out to the host. + # + # ⚠️ Do not bind-mount this path. Upstream pairs `StateDirectory=` with + # `DynamicUser=`, which makes systemd hold the state at + # `/var/lib/private/openbao` and symlink this to it; that relocation is a + # rename, and a rename of an active mount point fails `EBUSY` at + # `STATE_DIRECTORY` — the unit then dies before `bao` runs at all. stateDir = "/var/lib/openbao"; - hostStateDir = "/var/lib/swarm-bao"; + + # The TLS material is its own small bind, on both sides of the boundary at + # the same path. Separate from `stateDir` because a HOST unit writes it and + # the container only reads it, and because systemd owns `stateDir` and moves + # it around; this directory is ours. + tlsDir = "/var/lib/swarm-bao-tls"; + + # Where the key surfaces for openbao to open. NOT `${tlsDir}/server-key.pem`: + # that file is root-owned 0600 on the host, and the service runs as a + # `DynamicUser`, so it cannot read the bind-mounted original. `LoadCredential` + # is systemd's answer to exactly this — PID 1 reads the source as root and + # re-exposes it inside the unit, owned by the service's own account. + # `$CREDENTIALS_DIRECTORY` is `/run/credentials/`, and the config file + # is rendered ahead of time, so the path is spelled out rather than read from + # the environment. + serverKeyCredential = "server-key"; + serverKeyCredentialPath = "/run/credentials/openbao.service/${serverKeyCredential}"; # The PIN is deliberately absent here. It arrives as `BAO_HSM_PIN` from an # EnvironmentFile the provisioning unit writes, because a value interpolated @@ -55,8 +76,9 @@ let # empty is the difference between a store that unseals itself and one that # needs a human after every restart. # The PKCS11 token store and its PINs live on the HOST and are bind-mounted - # in. Losing them loses the sealed store, so they are a host-level fact an - # operator can back up — the same reasoning as the raft data below. + # in. Losing them loses the sealed store — the raft data is worth nothing + # without the key that unseals it — so unlike the state directory these are + # deliberately a host-level fact an operator can back up. tokenStoreDir = "/var/lib/swarm-bao-token"; pinEnvFile = "${tokenStoreDir}/pin.env"; @@ -73,12 +95,15 @@ let # sees, not a coercion error from here. domainBase = if swarmDomain == null then "invalid" else swarmDomain; - # Where the leaf lands for openbao to read. Inside the container, because - # `hostStateDir` is already bind-mounted at `stateDir` — so the delivery - # below needs no second mount, and nothing has to bind - # `deploy.hive-controller.tls.stateDir`, which holds the hive CA's private key. - serverCertPath = "${stateDir}/server.pem"; - serverKeyPath = "${stateDir}/server-key.pem"; + # Where the leaf lands for openbao to read. `tlsDir` is bind-mounted at the + # same path on both sides, so the delivery below needs no second mount, and + # nothing has to bind `deploy.hive-controller.tls.stateDir`, which holds the + # hive CA's private key. + # + # The certificate and the client CA are public material and are read straight + # off the mount; only the key takes the credential path above. + serverCertPath = "${tlsDir}/server.pem"; + serverKeyPath = "${tlsDir}/server-key.pem"; # The host-side sources, verbatim from the options — no fallback, because a # fallback is exactly the CA opinion this module must not hold. The units @@ -99,14 +124,14 @@ let # has said what to trust yet". listenerTls = { tls_cert_file = serverCertPath; - tls_key_file = serverKeyPath; + tls_key_file = serverKeyCredentialPath; } // lib.optionalAttrs (baoDeploy.clientCaFile != null) { tls_client_ca_file = clientCaPath; tls_require_and_verify_client_cert = true; }; - clientCaPath = "${stateDir}/client-ca.pem"; + clientCaPath = "${tlsDir}/client-ca.pem"; extraListeners = lib.listToAttrs ( lib.imap1 ( @@ -365,8 +390,8 @@ in # the container is ordered. Same trap `hostClientSecretDir` # documents in ./swarm-authelia.nix. Ordering against whatever # mints it belongs with whatever named the path, not here. - # 2. `hostStateDir` is already mounted at `stateDir`, so a copy needs - # no second mount. + # 2. `tlsDir` is already mounted at the same path inside, so a copy + # needs no second mount. # 3. A directory holding a leaf usually holds the CA's private key # beside it. Binding that directory to reach one file inside it # would hand the container authority to mint any name that CA can — @@ -382,7 +407,11 @@ in }; script = '' set -euo pipefail - install -d -m 0700 ${hostStateDir} + # 0755, not 0700: the container reads the certificate and the client + # CA straight off this mount as a non-root user, so it has to be able + # to traverse the directory. The key inside stays 0600 and reaches + # the service through `LoadCredential` instead. + install -d -m 0755 ${tlsDir} # Fail loudly rather than start a store that cannot serve. The path # is configured, so a missing file means whatever was supposed to @@ -397,8 +426,8 @@ in fi done - install -m 0644 ${lib.escapeShellArg serverCertSrc} ${hostStateDir}/server.pem - install -m 0600 ${lib.escapeShellArg serverKeySrc} ${hostStateDir}/server-key.pem + install -m 0644 ${lib.escapeShellArg serverCertSrc} ${tlsDir}/server.pem + install -m 0600 ${lib.escapeShellArg serverKeySrc} ${tlsDir}/server-key.pem '' + lib.optionalString (baoDeploy.clientCaFile != null) '' @@ -406,7 +435,7 @@ in echo "deploy.bao.clientCaFile names ${baoDeploy.clientCaFile}, which is missing or empty." >&2 exit 1 fi - install -m 0644 ${lib.escapeShellArg baoDeploy.clientCaFile} ${hostStateDir}/client-ca.pem + install -m 0644 ${lib.escapeShellArg baoDeploy.clientCaFile} ${tlsDir}/client-ca.pem ''; }; @@ -469,15 +498,16 @@ in # rather than a convenience for nginx. privateNetwork = false; - # Raft state outlives the container. `ephemeral = false` keeps the - # container's own /var, but a bind makes the store's data a host-level - # fact an operator can back up and a `nixos-container destroy` cannot - # take with it — which for the swarm's secrets is the difference between - # a rebuild and an outage. + # Only what a HOST unit writes and this container reads crosses the + # boundary. The raft state deliberately does not: `ephemeral = false` + # keeps the container's own /var, systemd owns `${stateDir}` through + # `StateDirectory=`, and binding over it is what breaks the unit. bindMounts = { - ${stateDir} = { - hostPath = hostStateDir; - isReadOnly = false; + # Read-only: `swarm-bao-certs` on the host is the only writer, and + # the store has no reason to modify its own identity. + ${tlsDir} = { + hostPath = tlsDir; + isReadOnly = true; }; } // lib.optionalAttrs (baoDeploy.seal == "pkcs11") { @@ -528,15 +558,30 @@ in // sealSettings; }; - # The PIN reaches openbao as an environment variable read from a - # 0400 file the provisioning unit wrote — never as a value in this - # expression, which would render it world-readable into the store. - # `TPM2_PKCS11_STORE` is required because the store is not at the - # library's default location. - systemd.services.openbao.serviceConfig = lib.mkIf (baoDeploy.seal == "pkcs11") { - EnvironmentFile = pinEnvFile; - Environment = [ "TPM2_PKCS11_STORE=${tokenStoreDir}" ]; - }; + # The private key crosses the user boundary here, not on the mount. + # `swarm-bao-certs` installs it 0600 root-owned, and the unit runs + # as a `DynamicUser`, so the bind-mounted file is unreadable to it — + # PID 1 opens the source as root and re-exposes it under + # `${serverKeyCredentialPath}`, owned by the service's own account. + # + # One assignment, not two: `serviceConfig.X = …` beside a + # `serviceConfig = …` is a duplicate attribute inside a single + # attrset literal and does not parse. Module merging happens across + # `config` blocks, not within a literal — so the seal's half joins + # with `optionalAttrs`. + systemd.services.openbao.serviceConfig = + lib.optionalAttrs haveServerTls { + LoadCredential = [ "${serverKeyCredential}:${serverKeyPath}" ]; + } + # The PIN reaches openbao as an environment variable read from a + # 0400 file the provisioning unit wrote — never as a value in this + # expression, which would render it world-readable into the store. + # `TPM2_PKCS11_STORE` is required because the store is not at the + # library's default location. + // lib.optionalAttrs (baoDeploy.seal == "pkcs11") { + EnvironmentFile = pinEnvFile; + Environment = [ "TPM2_PKCS11_STORE=${tokenStoreDir}" ]; + }; # ⚠️ Upstream sets `restartIfChanged = false` on this unit, on # purpose: a restart SEALS the store and disconnects every client.