docs/swarm/secrets.md

@@ -55,7 +55,7 @@ neither is a renaming of the other.
 | authelia subject store | `swarmctl` and `swarm-authelia-bridge` | `users.yml` — one file, read and written by both | `swarmctl`, on the host that runs authelia |
 | wireguard private key | **the operator** — `wg genkey` | whatever `swarm.wireguard.privateKeyFile` names | always operator-provided; nothing generates this for you |
 | queue auth-callout nkeys (user seed + account seed) | `swarm-nats-callout-keys` first-boot unit, when `nats.autoGenerateCallout` is set | `/var/lib/swarm-nats-callout/{callout-user,issuer}.seed`, `0600` | operator mints both with `nk` and names them in `nats.calloutUserSeedFile` / `nats.calloutIssuerSeedFile` |
-| the secret store's own contents | openbao, on first `bao operator init` — **an operator action, not a unit** | inside the `swarm-bao` container, at its own `/var/lib/openbao`, kept across rebuilds by `ephemeral = false`. ⚠️ **Not a host path**: `nixos-container destroy swarm-bao` takes the raft data with it, so back up the container's tree, not `/var/lib/`. Only the store's TLS material (`/var/lib/swarm-bao-tls`) and its PKCS11 token (`/var/lib/swarm-bao-token`) are host-level | n/a — there is one store; a hive elsewhere is a *client* of it and holds none of this |
+| the secret store's own contents | openbao, on first `bao operator init` — **an operator action, not a unit** | `/var/lib/swarm-bao` on the host of whoever runs the store, bind-mounted into the `swarm-bao` container | n/a — there is one store; a hive elsewhere is a *client* of it and holds none of this |
 | the secret store's unseal material | the HSM/TPM under `deploy.bao.seal = "pkcs11"`; openbao itself under `"shamir"` | in the token; or held by whoever ran `bao operator init`, which is what `"shamir"` means and why it is stated rather than inferred | n/a — only the host running the store seals anything |
 
 The three keys authelia mints for itself are generated in-container precisely

nix/host-modules/swarm-bao.nix

@@ -39,34 +39,13 @@ let
   networkCfg = hyperhiveCfg.network;
   swarmDomain = hyperhiveCfg.swarm.domain;
 
-  # Upstream's own default, kept so its documentation matches. The raft data
+  # Two names for one location. `stateDir` is where openbao writes inside the
-  # lives INSIDE the container on `ephemeral = false`, the same way
+  # container — upstream's own default, kept so its documentation matches. The
-  # ./swarm-grafana.nix keeps its sqlite database — no sibling service binds
+  # host path is a sibling of the other swarm services' state rather than a
-  # its state out to the host.
+  # path inside the container's tree, so `nixos-container destroy` cannot take
-  #
+  # the swarm's secrets with it.
-  # ⚠️ Do not bind-mount this path. Upstream pairs `StateDirectory=` with
-  # `DynamicUser=`, which makes systemd hold the state at
-  # `/var/lib/private/openbao` and symlink this to it; that relocation is a
-  # rename, and a rename of an active mount point fails `EBUSY` at
-  # `STATE_DIRECTORY` — the unit then dies before `bao` runs at all.
   stateDir = "/var/lib/openbao";
-
+  hostStateDir = "/var/lib/swarm-bao";
-  # The TLS material is its own small bind, on both sides of the boundary at
-  # the same path. Separate from `stateDir` because a HOST unit writes it and
-  # the container only reads it, and because systemd owns `stateDir` and moves
-  # it around; this directory is ours.
-  tlsDir = "/var/lib/swarm-bao-tls";
-
-  # Where the key surfaces for openbao to open. NOT `${tlsDir}/server-key.pem`:
-  # that file is root-owned 0600 on the host, and the service runs as a
-  # `DynamicUser`, so it cannot read the bind-mounted original. `LoadCredential`
-  # is systemd's answer to exactly this — PID 1 reads the source as root and
-  # re-exposes it inside the unit, owned by the service's own account.
-  # `$CREDENTIALS_DIRECTORY` is `/run/credentials/<unit>`, and the config file
-  # is rendered ahead of time, so the path is spelled out rather than read from
-  # the environment.
-  serverKeyCredential = "server-key";
-  serverKeyCredentialPath = "/run/credentials/openbao.service/${serverKeyCredential}";
 
   # The PIN is deliberately absent here. It arrives as `BAO_HSM_PIN` from an
   # EnvironmentFile the provisioning unit writes, because a value interpolated
@@ -76,9 +55,8 @@ let
   # empty is the difference between a store that unseals itself and one that
   # needs a human after every restart.
   # The PKCS11 token store and its PINs live on the HOST and are bind-mounted
-  # in. Losing them loses the sealed store — the raft data is worth nothing
+  # in. Losing them loses the sealed store, so they are a host-level fact an
-  # without the key that unseals it — so unlike the state directory these are
+  # operator can back up — the same reasoning as the raft data below.
-  # deliberately a host-level fact an operator can back up.
   tokenStoreDir = "/var/lib/swarm-bao-token";
   pinEnvFile = "${tokenStoreDir}/pin.env";
 
@@ -95,15 +73,12 @@ let
   # sees, not a coercion error from here.
   domainBase = if swarmDomain == null then "invalid" else swarmDomain;
 
-  # Where the leaf lands for openbao to read. `tlsDir` is bind-mounted at the
+  # Where the leaf lands for openbao to read. Inside the container, because
-  # same path on both sides, so the delivery below needs no second mount, and
+  # `hostStateDir` is already bind-mounted at `stateDir` — so the delivery
-  # nothing has to bind `deploy.hive-controller.tls.stateDir`, which holds the
+  # below needs no second mount, and nothing has to bind
-  # hive CA's private key.
+  # `deploy.hive-controller.tls.stateDir`, which holds the hive CA's private key.
-  #
+  serverCertPath = "${stateDir}/server.pem";
-  # The certificate and the client CA are public material and are read straight
+  serverKeyPath = "${stateDir}/server-key.pem";
-  # off the mount; only the key takes the credential path above.
-  serverCertPath = "${tlsDir}/server.pem";
-  serverKeyPath = "${tlsDir}/server-key.pem";
 
   # The host-side sources, verbatim from the options — no fallback, because a
   # fallback is exactly the CA opinion this module must not hold. The units
@@ -124,14 +99,14 @@ let
   # has said what to trust yet".
   listenerTls = {
     tls_cert_file = serverCertPath;
-    tls_key_file = serverKeyCredentialPath;
+    tls_key_file = serverKeyPath;
   }
   // lib.optionalAttrs (baoDeploy.clientCaFile != null) {
     tls_client_ca_file = clientCaPath;
     tls_require_and_verify_client_cert = true;
   };
 
-  clientCaPath = "${tlsDir}/client-ca.pem";
+  clientCaPath = "${stateDir}/client-ca.pem";
 
   extraListeners = lib.listToAttrs (
     lib.imap1 (
@@ -390,8 +365,8 @@ in
       #      the container is ordered. Same trap `hostClientSecretDir`
       #      documents in ./swarm-authelia.nix. Ordering against whatever
       #      mints it belongs with whatever named the path, not here.
-      #   2. `tlsDir` is already mounted at the same path inside, so a copy
+      #   2. `hostStateDir` is already mounted at `stateDir`, so a copy needs
-      #      needs no second mount.
+      #      no second mount.
       #   3. A directory holding a leaf usually holds the CA's private key
       #      beside it. Binding that directory to reach one file inside it
       #      would hand the container authority to mint any name that CA can —
@@ -407,11 +382,7 @@ in
         };
         script = ''
           set -euo pipefail
-          # 0755, not 0700: the container reads the certificate and the client
+          install -d -m 0700 ${hostStateDir}
-          # CA straight off this mount as a non-root user, so it has to be able
-          # to traverse the directory. The key inside stays 0600 and reaches
-          # the service through `LoadCredential` instead.
-          install -d -m 0755 ${tlsDir}
 
           # Fail loudly rather than start a store that cannot serve. The path
           # is configured, so a missing file means whatever was supposed to
@@ -426,8 +397,8 @@ in
             fi
           done
 
-          install -m 0644 ${lib.escapeShellArg serverCertSrc} ${tlsDir}/server.pem
+          install -m 0644 ${lib.escapeShellArg serverCertSrc} ${hostStateDir}/server.pem
-          install -m 0600 ${lib.escapeShellArg serverKeySrc} ${tlsDir}/server-key.pem
+          install -m 0600 ${lib.escapeShellArg serverKeySrc} ${hostStateDir}/server-key.pem
         ''
         + lib.optionalString (baoDeploy.clientCaFile != null) ''
 
@@ -435,7 +406,7 @@ in
             echo "deploy.bao.clientCaFile names ${baoDeploy.clientCaFile}, which is missing or empty." >&2
             exit 1
           fi
-          install -m 0644 ${lib.escapeShellArg baoDeploy.clientCaFile} ${tlsDir}/client-ca.pem
+          install -m 0644 ${lib.escapeShellArg baoDeploy.clientCaFile} ${hostStateDir}/client-ca.pem
         '';
       };
 
@@ -498,16 +469,15 @@ in
         # rather than a convenience for nginx.
         privateNetwork = false;
 
-        # Only what a HOST unit writes and this container reads crosses the
+        # Raft state outlives the container. `ephemeral = false` keeps the
-        # boundary. The raft state deliberately does not: `ephemeral = false`
+        # container's own /var, but a bind makes the store's data a host-level
-        # keeps the container's own /var, systemd owns `${stateDir}` through
+        # fact an operator can back up and a `nixos-container destroy` cannot
-        # `StateDirectory=`, and binding over it is what breaks the unit.
+        # take with it — which for the swarm's secrets is the difference between
+        # a rebuild and an outage.
         bindMounts = {
-          # Read-only: `swarm-bao-certs` on the host is the only writer, and
+          ${stateDir} = {
-          # the store has no reason to modify its own identity.
+            hostPath = hostStateDir;
-          ${tlsDir} = {
+            isReadOnly = false;
-            hostPath = tlsDir;
-            isReadOnly = true;
           };
         }
         // lib.optionalAttrs (baoDeploy.seal == "pkcs11") {
@@ -558,30 +528,15 @@ in
               // sealSettings;
             };
 
-            # The private key crosses the user boundary here, not on the mount.
+            # The PIN reaches openbao as an environment variable read from a
-            # `swarm-bao-certs` installs it 0600 root-owned, and the unit runs
+            # 0400 file the provisioning unit wrote — never as a value in this
-            # as a `DynamicUser`, so the bind-mounted file is unreadable to it —
+            # expression, which would render it world-readable into the store.
-            # PID 1 opens the source as root and re-exposes it under
+            # `TPM2_PKCS11_STORE` is required because the store is not at the
-            # `${serverKeyCredentialPath}`, owned by the service's own account.
+            # library's default location.
-            #
+            systemd.services.openbao.serviceConfig = lib.mkIf (baoDeploy.seal == "pkcs11") {
-            # One assignment, not two: `serviceConfig.X = …` beside a
+              EnvironmentFile = pinEnvFile;
-            # `serviceConfig = …` is a duplicate attribute inside a single
+              Environment = [ "TPM2_PKCS11_STORE=${tokenStoreDir}" ];
-            # attrset literal and does not parse. Module merging happens across
+            };
-            # `config` blocks, not within a literal — so the seal's half joins
-            # with `optionalAttrs`.
-            systemd.services.openbao.serviceConfig =
-              lib.optionalAttrs haveServerTls {
-                LoadCredential = [ "${serverKeyCredential}:${serverKeyPath}" ];
-              }
-              # The PIN reaches openbao as an environment variable read from a
-              # 0400 file the provisioning unit wrote — never as a value in this
-              # expression, which would render it world-readable into the store.
-              # `TPM2_PKCS11_STORE` is required because the store is not at the
-              # library's default location.
-              // lib.optionalAttrs (baoDeploy.seal == "pkcs11") {
-                EnvironmentFile = pinEnvFile;
-                Environment = [ "TPM2_PKCS11_STORE=${tokenStoreDir}" ];
-              };
 
             # ⚠️ Upstream sets `restartIfChanged = false` on this unit, on
             # purpose: a restart SEALS the store and disconnects every client.