give cert-auth role tokens a 15m ttl so logins stop leaving 32-day leases

docs/swarm/credentials.md

@@ -454,8 +454,21 @@ The roles of `swarm-nats-auth`, `swarm-logs-auth` and `swarm-metrics-auth` issue
 batch tokens with a five-minute TTL (`token_type=batch`, `token_ttl=5m`). The
 responders log in on every check, and the store keeps no record of a batch token,
 so unauthenticated requests don't grow its token storage. Nothing can renew
-or revoke a batch token. The other roles keep service tokens, because some of
+or revoke a batch token.
-their readers hold a handle for longer than five minutes.
+
+The other roles issue service tokens that live 15 minutes (`token_ttl=15m` and
+`token_max_ttl=15m`). That covers the roles `swarm-bao.nix` writes, and the
+agent and hive roles the controller writes through
+`SecretStore::write_cert_role`, which takes the same value from
+`CERT_ROLE_TOKEN_TTL`. The store keeps a lease for every token until it
+expires, and the mount default of 32 days leaves one lease per login for a
+month; at the 17 thousand logins a day this swarm made, that came to half a
+million leases. A reader logs in per operation and holds the token for seconds.
+The longest hold is the controller's forge work, which stops after two minutes.
+A client that keeps a `SecretStore` handle longer than 15 minutes must log in
+again, because nothing renews the token. A role the controller writes takes the
+TTL at its next write: a hive's at the controller's start, an agent's at its
+next certificate renewal.
 
 ### What a missing leaf costs
 

nix/host-modules/swarm-bao.nix

@@ -988,6 +988,15 @@ let
   # live agent's leaf once it is past half of it (`agent_renewal.rs`).
   agentPkiLeafTtl = "168h";
 
+  # Lifetime of a token from a cert login, on every cert-auth role this file
+  # writes except the batch ones. The store keeps a lease per token until it
+  # expires; a login per operation at the mount default (32 days) leaves one
+  # lease each for a month. Every holder logs in, works and drops the token
+  # within two minutes (the controller's forge work is cut off at
+  # `FORGE_DEADLINE`). Rust-written roles take `CERT_ROLE_TOKEN_TTL`, which
+  # is the same value.
+  certRoleTokenTtl = "15m";
+
   # The agent CA's certificate, cached on this host by `swarm-bao-agent-pki`,
   # so `swarm-bao-certs` can compose the listener's bundle before the store
   # is up. Public material.
@@ -1372,15 +1381,18 @@ let
   readerRoleWrite =
     obj:
     let
-      batchArgs = lib.optionalString (obj.batchToken or false
+      ttlArgs =
-      ) " \\\n  token_type=batch \\\n  token_ttl=5m";
+        if obj.batchToken or false then
+          " \\\n  token_type=batch \\\n  token_ttl=5m"
+        else
+          " \\\n  token_ttl=${certRoleTokenTtl} \\\n  token_max_ttl=${certRoleTokenTtl}";
     in
     ''
       bao write auth/cert/certs/${lib.escapeShellArg obj.name} \
         certificate=@${tlsDir}/client-ca.pem \
         allowed_common_names=${lib.escapeShellArg obj.cn} \
         token_policies=${lib.escapeShellArg obj.name} \
-        display_name=${lib.escapeShellArg obj.cn}${batchArgs}
+        display_name=${lib.escapeShellArg obj.cn}${ttlArgs}
     '';
 
   # A unit that logs in as the granter and runs `body`, ordered after the
@@ -3660,7 +3672,9 @@ in
             certificate=@${tlsDir}/client-ca.pem \
             allowed_common_names=${lib.escapeShellArg controllerCn} \
             token_policies=${lib.escapeShellArg controllerPolicyName} \
-            display_name=${lib.escapeShellArg controllerCn}
+            display_name=${lib.escapeShellArg controllerCn} \
+            token_ttl=${certRoleTokenTtl} \
+            token_max_ttl=${certRoleTokenTtl}
         '';
       };
 
@@ -3684,7 +3698,9 @@ in
           certificate=@${tlsDir}/client-ca.pem \
           allowed_common_names=${lib.escapeShellArg secretPublisherCn} \
           token_policies=${lib.escapeShellArg secretPublisherPolicyName} \
-          display_name=${lib.escapeShellArg secretPublisherCn}
+          display_name=${lib.escapeShellArg secretPublisherCn} \
+          token_ttl=${certRoleTokenTtl} \
+          token_max_ttl=${certRoleTokenTtl}
       '';
 
       # A THIRD sibling, for the reason the second one's comment gives: these
@@ -3703,7 +3719,9 @@ in
           certificate=@${tlsDir}/client-ca.pem \
           allowed_common_names=${lib.escapeShellArg matrixCtlCn} \
           token_policies=${lib.escapeShellArg matrixCtlPolicyName} \
-          display_name=${lib.escapeShellArg matrixCtlCn}
+          display_name=${lib.escapeShellArg matrixCtlCn} \
+          token_ttl=${certRoleTokenTtl} \
+          token_max_ttl=${certRoleTokenTtl}
       '';
 
       # 🩸 The principals that each log in with a leaf of their own, not as the
@@ -3757,7 +3775,9 @@ in
           certificate=@${tlsDir}/client-ca.pem \
           allowed_common_names=${lib.escapeShellArg servicesIssuerCn} \
           token_policies=${lib.escapeShellArg servicesIssuerPolicyName} \
-          display_name=${lib.escapeShellArg servicesIssuerCn}
+          display_name=${lib.escapeShellArg servicesIssuerCn} \
+          token_ttl=${certRoleTokenTtl} \
+          token_max_ttl=${certRoleTokenTtl}
       '';
 
       # A FIFTH sibling: the queue's issuing role, its policy and its login
@@ -3793,7 +3813,9 @@ in
           certificate=@${tlsDir}/client-ca.pem \
           allowed_common_names=${lib.escapeShellArg natsCn} \
           token_policies=${lib.escapeShellArg natsPolicyName} \
-          display_name=${lib.escapeShellArg natsCn}
+          display_name=${lib.escapeShellArg natsCn} \
+          token_ttl=${certRoleTokenTtl} \
+          token_max_ttl=${certRoleTokenTtl}
       '';
 
       # A SIXTH sibling, the nats unit's shape once per hive: each hive's
@@ -3829,7 +3851,9 @@ in
                 certificate=@${tlsDir}/client-ca.pem \
                 allowed_common_names=${lib.escapeShellArg obj.cn} \
                 token_policies=${lib.escapeShellArg obj.name} \
-                display_name=${lib.escapeShellArg obj.cn}
+                display_name=${lib.escapeShellArg obj.cn} \
+                token_ttl=${certRoleTokenTtl} \
+                token_max_ttl=${certRoleTokenTtl}
             '') hiveLeafIssuers
           );
 

nix/module-eval/bao-grants.nix

@@ -885,9 +885,9 @@ let
     {
       # The three agent-token checkers log in once per check; a batch token
       # leaves nothing stored per request. The CI runner's role, written by the
-      # same helper, keeps service tokens, and still carries its policy, so a
+      # same helper, keeps service tokens with the short TTL, and still carries
-      # misspelled script name cannot make the absence pass.
+      # its policy, so a misspelled script name cannot make the absence pass.
-      name = "the agent-token checkers' roles issue batch tokens and a holder's role does not";
+      name = "the agent-token checkers' roles issue batch tokens and the CI runner's issues short service tokens";
       ok =
         let
           script = unit: baoGrantHere.systemd.services.${unit}.script;
@@ -907,7 +907,8 @@ let
         && batch "swarm-bao-metrics-auth-policy" "swarm-metrics-auth"
         && lib.hasInfix "token_policies=swarm-ci-runner" ci
         && !(lib.hasInfix "token_type" ci)
-        && !(lib.hasInfix "token_ttl" ci);
+        && lib.hasInfix "token_ttl=15m" ci
+        && lib.hasInfix "token_max_ttl=15m" ci;
     }
     {
       name = "the PKI unit signs the metrics store responder's leaf under its own subject";
@@ -1471,6 +1472,59 @@ let
         && !(lib.hasPrefix "swarm-" cn)
         && !(lib.hasPrefix "hive-" cn);
     }
+    {
+      # A cert login leaves a lease until its token expires, and a role without
+      # `token_ttl` gets the mount's default of 32 days. Every role write in
+      # every unit is read, so the TTL must be on the command itself, not
+      # somewhere else in the script. A role is a service token at `15m`, or a
+      # batch token at `5m`.
+      #
+      # The presence control: the roles found include the controller's, the
+      # granter's, a batch responder's, the CI runner's and a per-hive leaf
+      # role, so a changed spelling of the write cannot leave the loop with
+      # nothing to check.
+      name = "every cert-auth role the units write issues short-lived tokens";
+      ok =
+        let
+          roleWrites =
+            cfg:
+            lib.concatMap (
+              u:
+              let
+                chunks = lib.tail (lib.splitString "bao write auth/cert/certs/" u.script);
+                commandOf =
+                  chunk:
+                  let
+                    ls = lib.splitString "\n" chunk;
+                    last = lib.lists.findFirstIndex (l: !(lib.hasSuffix "\\" l)) 0 ls;
+                  in
+                  lib.concatStringsSep "\n" (lib.take (last + 1) ls);
+              in
+              map (c: {
+                role = lib.head (lib.splitString " " c);
+                command = commandOf c;
+              }) chunks
+            ) (lib.attrValues cfg.systemd.services);
+          writes = roleWrites baoGrantHere ++ roleWrites baoGrantTwoHives;
+          short =
+            w:
+            (lib.hasInfix "token_ttl=15m" w.command && lib.hasInfix "token_max_ttl=15m" w.command)
+            || (
+              lib.hasInfix "token_type=batch" w.command
+              && lib.hasInfix "token_ttl=5m" w.command
+              && !(lib.hasInfix "token_max_ttl" w.command)
+            );
+          roles = map (w: w.role) writes;
+        in
+        lib.all short writes
+        && lib.all (r: lib.elem r roles) [
+          "swarm-controller"
+          "bao-granter"
+          "swarm-nats-auth"
+          "swarm-ci-runner"
+          "swarm-hive-leaf-h2"
+        ];
+    }
     {
       # The other principals are what they were: no unit but the granter's own
       # hands its policy to a role, and none of them logs in as it.

swarm-secret-client/README.md

@@ -84,5 +84,6 @@ It has no opinion on **what** a caller may read. That is the policy attached to
 the cert role, and it lives in the store.
 
 It holds the token minted at login and renews nothing. A handle is built per
-credential, so the login is the cheap part of a rare operation — a caller that
+credential, so the login is the cheap part of a rare operation. Cert-auth roles
-wanted to keep one alive across a token's lifetime would need more than this.
+mint tokens for [`client::CERT_ROLE_TOKEN_TTL`]; a caller that wanted to keep one
+alive longer would need more than this.

swarm-secret-client/src/client.rs

@@ -5,7 +5,10 @@ use std::time::Duration;
 use rustify_derive::Endpoint;
 use serde::{Deserialize, Serialize, de::DeserializeOwned};
 use vaultrs::{
-    api::pki::{requests::GenerateCertificateRequest, responses::GenerateCertificateResponse},
+    api::{
+        auth::cert::requests::CreateCaCertificateRoleRequest,
+        pki::{requests::GenerateCertificateRequest, responses::GenerateCertificateResponse},
+    },
     client::{Client, VaultClient, VaultClientSettings, VaultClientSettingsBuilder},
 };
 
@@ -25,6 +28,15 @@ pub const ENV_CACERT: &str = "BAO_CACERT";
 /// The auth mount a cert login goes through, unless a caller says otherwise.
 pub const DEFAULT_CERT_MOUNT: &str = "cert";
 
+/// How long a token from a cert login lives, and the longest it can be made to.
+///
+/// Every cert-auth role is written with it as `token_ttl` and `token_max_ttl`.
+/// The store keeps a lease for each token until it expires, so a role without
+/// it leaves one lease per login for the mount's default of 32 days.
+/// [`SecretStore`] holds the token it logs in with and renews nothing: log in
+/// again rather than keep a handle past this.
+pub const CERT_ROLE_TOKEN_TTL: &str = "15m";
+
 /// Upper bound on one request to the store, login included.
 ///
 /// vaultrs sets no timeout by default, so a store that accepts the connection
@@ -371,18 +383,8 @@ impl SecretStore {
         common_name: &str,
         policies: &[&str],
     ) -> Result<(), Error> {
-        let mut opts =
+        let request = cert_role_request(mount, name, certificate, common_name, policies);
-            vaultrs::api::auth::cert::requests::CreateCaCertificateRoleRequest::builder();
+        vaultrs::api::exec_with_empty(&self.inner, request).await?;
-        opts.allowed_common_names(vec![common_name.to_owned()])
-            .token_policies(policies.iter().map(|p| (*p).to_owned()).collect::<Vec<_>>());
-        vaultrs::auth::cert::ca_cert_role::set(
-            &self.inner,
-            mount,
-            name,
-            certificate,
-            Some(&mut opts),
-        )
-        .await?;
         Ok(())
     }
 
@@ -667,6 +669,28 @@ fn issue_request(mount: &str, role: &str, common_name: &str) -> GenerateCertific
     }
 }
 
+/// The request that writes a cert-auth role: `certificate` pinned, only
+/// `common_name` accepted, `policies` attached, tokens living
+/// [`CERT_ROLE_TOKEN_TTL`].
+fn cert_role_request(
+    mount: &str,
+    name: &str,
+    certificate: &str,
+    common_name: &str,
+    policies: &[&str],
+) -> CreateCaCertificateRoleRequest {
+    CreateCaCertificateRoleRequest {
+        mount: mount.to_owned(),
+        name: name.to_owned(),
+        certificate: certificate.to_owned(),
+        allowed_common_names: Some(vec![common_name.to_owned()]),
+        token_policies: Some(policies.iter().map(|p| (*p).to_owned()).collect()),
+        token_ttl: Some(CERT_ROLE_TOKEN_TTL.to_owned()),
+        token_max_ttl: Some(CERT_ROLE_TOKEN_TTL.to_owned()),
+        ..CreateCaCertificateRoleRequest::default()
+    }
+}
+
 /// The store's answer, moved straight into the redacting
 /// [`mtls::Credential`]. `GenerateCertificateResponse` derives `Debug` and
 /// holds the private key, so it is consumed here and never formatted.
@@ -711,6 +735,20 @@ mod tests {
         assert_eq!(built.timeout, Some(VAULT_REQUEST_TIMEOUT));
     }
 
+    #[test]
+    fn a_cert_role_is_written_with_short_lived_tokens() {
+        let r = cert_role_request("cert", "hive-agent-a", "PEM", "hive-agent-a", &["shared"]);
+        assert_eq!(r.token_ttl.as_deref(), Some("15m"));
+        assert_eq!(r.token_max_ttl.as_deref(), Some("15m"));
+        // The control: the rest of the role is still written.
+        assert_eq!(
+            r.allowed_common_names,
+            Some(vec!["hive-agent-a".to_owned()])
+        );
+        assert_eq!(r.token_policies, Some(vec!["shared".to_owned()]));
+        assert_eq!(r.certificate, "PEM");
+    }
+
     #[test]
     fn a_complete_environment_is_accepted() {
         // The control: without this, every assertion below could be passing