From 00df3ccffb7303d2f50b38f37de2363a013f17fa Mon Sep 17 00:00:00 2001 From: atlas Date: Wed, 28 Oct 2026 18:32:44 +0100 Subject: [PATCH] give cert-auth role tokens a 15m ttl so logins stop leaving 32-day leases --- docs/swarm/credentials.md | 17 +++++++- nix/host-modules/swarm-bao.nix | 42 +++++++++++++++----- nix/module-eval/bao-grants.nix | 62 ++++++++++++++++++++++++++++-- swarm-secret-client/README.md | 5 ++- swarm-secret-client/src/client.rs | 64 ++++++++++++++++++++++++------- 5 files changed, 160 insertions(+), 30 deletions(-) diff --git a/docs/swarm/credentials.md b/docs/swarm/credentials.md index b04973e0..997e0fad 100644 --- a/docs/swarm/credentials.md +++ b/docs/swarm/credentials.md @@ -454,8 +454,21 @@ The roles of `swarm-nats-auth`, `swarm-logs-auth` and `swarm-metrics-auth` issue batch tokens with a five-minute TTL (`token_type=batch`, `token_ttl=5m`). The responders log in on every check, and the store keeps no record of a batch token, so unauthenticated requests don't grow its token storage. Nothing can renew -or revoke a batch token. The other roles keep service tokens, because some of -their readers hold a handle for longer than five minutes. +or revoke a batch token. + +The other roles issue service tokens that live 15 minutes (`token_ttl=15m` and +`token_max_ttl=15m`). That covers the roles `swarm-bao.nix` writes, and the +agent and hive roles the controller writes through +`SecretStore::write_cert_role`, which takes the same value from +`CERT_ROLE_TOKEN_TTL`. The store keeps a lease for every token until it +expires, and the mount default of 32 days leaves one lease per login for a +month; at the 17 thousand logins a day this swarm made, that came to half a +million leases. A reader logs in per operation and holds the token for seconds. +The longest hold is the controller's forge work, which stops after two minutes. +A client that keeps a `SecretStore` handle longer than 15 minutes must log in +again, because nothing renews the token. A role the controller writes takes the +TTL at its next write: a hive's at the controller's start, an agent's at its +next certificate renewal. ### What a missing leaf costs diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index b7f617e0..02f82ec2 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -988,6 +988,15 @@ let # live agent's leaf once it is past half of it (`agent_renewal.rs`). agentPkiLeafTtl = "168h"; + # Lifetime of a token from a cert login, on every cert-auth role this file + # writes except the batch ones. The store keeps a lease per token until it + # expires; a login per operation at the mount default (32 days) leaves one + # lease each for a month. Every holder logs in, works and drops the token + # within two minutes (the controller's forge work is cut off at + # `FORGE_DEADLINE`). Rust-written roles take `CERT_ROLE_TOKEN_TTL`, which + # is the same value. + certRoleTokenTtl = "15m"; + # The agent CA's certificate, cached on this host by `swarm-bao-agent-pki`, # so `swarm-bao-certs` can compose the listener's bundle before the store # is up. Public material. @@ -1372,15 +1381,18 @@ let readerRoleWrite = obj: let - batchArgs = lib.optionalString (obj.batchToken or false - ) " \\\n token_type=batch \\\n token_ttl=5m"; + ttlArgs = + if obj.batchToken or false then + " \\\n token_type=batch \\\n token_ttl=5m" + else + " \\\n token_ttl=${certRoleTokenTtl} \\\n token_max_ttl=${certRoleTokenTtl}"; in '' bao write auth/cert/certs/${lib.escapeShellArg obj.name} \ certificate=@${tlsDir}/client-ca.pem \ allowed_common_names=${lib.escapeShellArg obj.cn} \ token_policies=${lib.escapeShellArg obj.name} \ - display_name=${lib.escapeShellArg obj.cn}${batchArgs} + display_name=${lib.escapeShellArg obj.cn}${ttlArgs} ''; # A unit that logs in as the granter and runs `body`, ordered after the @@ -3660,7 +3672,9 @@ in certificate=@${tlsDir}/client-ca.pem \ allowed_common_names=${lib.escapeShellArg controllerCn} \ token_policies=${lib.escapeShellArg controllerPolicyName} \ - display_name=${lib.escapeShellArg controllerCn} + display_name=${lib.escapeShellArg controllerCn} \ + token_ttl=${certRoleTokenTtl} \ + token_max_ttl=${certRoleTokenTtl} ''; }; @@ -3684,7 +3698,9 @@ in certificate=@${tlsDir}/client-ca.pem \ allowed_common_names=${lib.escapeShellArg secretPublisherCn} \ token_policies=${lib.escapeShellArg secretPublisherPolicyName} \ - display_name=${lib.escapeShellArg secretPublisherCn} + display_name=${lib.escapeShellArg secretPublisherCn} \ + token_ttl=${certRoleTokenTtl} \ + token_max_ttl=${certRoleTokenTtl} ''; # A THIRD sibling, for the reason the second one's comment gives: these @@ -3703,7 +3719,9 @@ in certificate=@${tlsDir}/client-ca.pem \ allowed_common_names=${lib.escapeShellArg matrixCtlCn} \ token_policies=${lib.escapeShellArg matrixCtlPolicyName} \ - display_name=${lib.escapeShellArg matrixCtlCn} + display_name=${lib.escapeShellArg matrixCtlCn} \ + token_ttl=${certRoleTokenTtl} \ + token_max_ttl=${certRoleTokenTtl} ''; # 🩸 The principals that each log in with a leaf of their own, not as the @@ -3757,7 +3775,9 @@ in certificate=@${tlsDir}/client-ca.pem \ allowed_common_names=${lib.escapeShellArg servicesIssuerCn} \ token_policies=${lib.escapeShellArg servicesIssuerPolicyName} \ - display_name=${lib.escapeShellArg servicesIssuerCn} + display_name=${lib.escapeShellArg servicesIssuerCn} \ + token_ttl=${certRoleTokenTtl} \ + token_max_ttl=${certRoleTokenTtl} ''; # A FIFTH sibling: the queue's issuing role, its policy and its login @@ -3793,7 +3813,9 @@ in certificate=@${tlsDir}/client-ca.pem \ allowed_common_names=${lib.escapeShellArg natsCn} \ token_policies=${lib.escapeShellArg natsPolicyName} \ - display_name=${lib.escapeShellArg natsCn} + display_name=${lib.escapeShellArg natsCn} \ + token_ttl=${certRoleTokenTtl} \ + token_max_ttl=${certRoleTokenTtl} ''; # A SIXTH sibling, the nats unit's shape once per hive: each hive's @@ -3829,7 +3851,9 @@ in certificate=@${tlsDir}/client-ca.pem \ allowed_common_names=${lib.escapeShellArg obj.cn} \ token_policies=${lib.escapeShellArg obj.name} \ - display_name=${lib.escapeShellArg obj.cn} + display_name=${lib.escapeShellArg obj.cn} \ + token_ttl=${certRoleTokenTtl} \ + token_max_ttl=${certRoleTokenTtl} '') hiveLeafIssuers ); diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index b6e48f1e..0aff4049 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -885,9 +885,9 @@ let { # The three agent-token checkers log in once per check; a batch token # leaves nothing stored per request. The CI runner's role, written by the - # same helper, keeps service tokens, and still carries its policy, so a - # misspelled script name cannot make the absence pass. - name = "the agent-token checkers' roles issue batch tokens and a holder's role does not"; + # same helper, keeps service tokens with the short TTL, and still carries + # its policy, so a misspelled script name cannot make the absence pass. + name = "the agent-token checkers' roles issue batch tokens and the CI runner's issues short service tokens"; ok = let script = unit: baoGrantHere.systemd.services.${unit}.script; @@ -907,7 +907,8 @@ let && batch "swarm-bao-metrics-auth-policy" "swarm-metrics-auth" && lib.hasInfix "token_policies=swarm-ci-runner" ci && !(lib.hasInfix "token_type" ci) - && !(lib.hasInfix "token_ttl" ci); + && lib.hasInfix "token_ttl=15m" ci + && lib.hasInfix "token_max_ttl=15m" ci; } { name = "the PKI unit signs the metrics store responder's leaf under its own subject"; @@ -1471,6 +1472,59 @@ let && !(lib.hasPrefix "swarm-" cn) && !(lib.hasPrefix "hive-" cn); } + { + # A cert login leaves a lease until its token expires, and a role without + # `token_ttl` gets the mount's default of 32 days. Every role write in + # every unit is read, so the TTL must be on the command itself, not + # somewhere else in the script. A role is a service token at `15m`, or a + # batch token at `5m`. + # + # The presence control: the roles found include the controller's, the + # granter's, a batch responder's, the CI runner's and a per-hive leaf + # role, so a changed spelling of the write cannot leave the loop with + # nothing to check. + name = "every cert-auth role the units write issues short-lived tokens"; + ok = + let + roleWrites = + cfg: + lib.concatMap ( + u: + let + chunks = lib.tail (lib.splitString "bao write auth/cert/certs/" u.script); + commandOf = + chunk: + let + ls = lib.splitString "\n" chunk; + last = lib.lists.findFirstIndex (l: !(lib.hasSuffix "\\" l)) 0 ls; + in + lib.concatStringsSep "\n" (lib.take (last + 1) ls); + in + map (c: { + role = lib.head (lib.splitString " " c); + command = commandOf c; + }) chunks + ) (lib.attrValues cfg.systemd.services); + writes = roleWrites baoGrantHere ++ roleWrites baoGrantTwoHives; + short = + w: + (lib.hasInfix "token_ttl=15m" w.command && lib.hasInfix "token_max_ttl=15m" w.command) + || ( + lib.hasInfix "token_type=batch" w.command + && lib.hasInfix "token_ttl=5m" w.command + && !(lib.hasInfix "token_max_ttl" w.command) + ); + roles = map (w: w.role) writes; + in + lib.all short writes + && lib.all (r: lib.elem r roles) [ + "swarm-controller" + "bao-granter" + "swarm-nats-auth" + "swarm-ci-runner" + "swarm-hive-leaf-h2" + ]; + } { # The other principals are what they were: no unit but the granter's own # hands its policy to a role, and none of them logs in as it. diff --git a/swarm-secret-client/README.md b/swarm-secret-client/README.md index 859d09d8..e6dd8595 100644 --- a/swarm-secret-client/README.md +++ b/swarm-secret-client/README.md @@ -84,5 +84,6 @@ It has no opinion on **what** a caller may read. That is the policy attached to the cert role, and it lives in the store. It holds the token minted at login and renews nothing. A handle is built per -credential, so the login is the cheap part of a rare operation — a caller that -wanted to keep one alive across a token's lifetime would need more than this. +credential, so the login is the cheap part of a rare operation. Cert-auth roles +mint tokens for [`client::CERT_ROLE_TOKEN_TTL`]; a caller that wanted to keep one +alive longer would need more than this. diff --git a/swarm-secret-client/src/client.rs b/swarm-secret-client/src/client.rs index 5fb4592d..778ece34 100644 --- a/swarm-secret-client/src/client.rs +++ b/swarm-secret-client/src/client.rs @@ -5,7 +5,10 @@ use std::time::Duration; use rustify_derive::Endpoint; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use vaultrs::{ - api::pki::{requests::GenerateCertificateRequest, responses::GenerateCertificateResponse}, + api::{ + auth::cert::requests::CreateCaCertificateRoleRequest, + pki::{requests::GenerateCertificateRequest, responses::GenerateCertificateResponse}, + }, client::{Client, VaultClient, VaultClientSettings, VaultClientSettingsBuilder}, }; @@ -25,6 +28,15 @@ pub const ENV_CACERT: &str = "BAO_CACERT"; /// The auth mount a cert login goes through, unless a caller says otherwise. pub const DEFAULT_CERT_MOUNT: &str = "cert"; +/// How long a token from a cert login lives, and the longest it can be made to. +/// +/// Every cert-auth role is written with it as `token_ttl` and `token_max_ttl`. +/// The store keeps a lease for each token until it expires, so a role without +/// it leaves one lease per login for the mount's default of 32 days. +/// [`SecretStore`] holds the token it logs in with and renews nothing: log in +/// again rather than keep a handle past this. +pub const CERT_ROLE_TOKEN_TTL: &str = "15m"; + /// Upper bound on one request to the store, login included. /// /// vaultrs sets no timeout by default, so a store that accepts the connection @@ -371,18 +383,8 @@ impl SecretStore { common_name: &str, policies: &[&str], ) -> Result<(), Error> { - let mut opts = - vaultrs::api::auth::cert::requests::CreateCaCertificateRoleRequest::builder(); - opts.allowed_common_names(vec![common_name.to_owned()]) - .token_policies(policies.iter().map(|p| (*p).to_owned()).collect::>()); - vaultrs::auth::cert::ca_cert_role::set( - &self.inner, - mount, - name, - certificate, - Some(&mut opts), - ) - .await?; + let request = cert_role_request(mount, name, certificate, common_name, policies); + vaultrs::api::exec_with_empty(&self.inner, request).await?; Ok(()) } @@ -667,6 +669,28 @@ fn issue_request(mount: &str, role: &str, common_name: &str) -> GenerateCertific } } +/// The request that writes a cert-auth role: `certificate` pinned, only +/// `common_name` accepted, `policies` attached, tokens living +/// [`CERT_ROLE_TOKEN_TTL`]. +fn cert_role_request( + mount: &str, + name: &str, + certificate: &str, + common_name: &str, + policies: &[&str], +) -> CreateCaCertificateRoleRequest { + CreateCaCertificateRoleRequest { + mount: mount.to_owned(), + name: name.to_owned(), + certificate: certificate.to_owned(), + allowed_common_names: Some(vec![common_name.to_owned()]), + token_policies: Some(policies.iter().map(|p| (*p).to_owned()).collect()), + token_ttl: Some(CERT_ROLE_TOKEN_TTL.to_owned()), + token_max_ttl: Some(CERT_ROLE_TOKEN_TTL.to_owned()), + ..CreateCaCertificateRoleRequest::default() + } +} + /// The store's answer, moved straight into the redacting /// [`mtls::Credential`]. `GenerateCertificateResponse` derives `Debug` and /// holds the private key, so it is consumed here and never formatted. @@ -711,6 +735,20 @@ mod tests { assert_eq!(built.timeout, Some(VAULT_REQUEST_TIMEOUT)); } + #[test] + fn a_cert_role_is_written_with_short_lived_tokens() { + let r = cert_role_request("cert", "hive-agent-a", "PEM", "hive-agent-a", &["shared"]); + assert_eq!(r.token_ttl.as_deref(), Some("15m")); + assert_eq!(r.token_max_ttl.as_deref(), Some("15m")); + // The control: the rest of the role is still written. + assert_eq!( + r.allowed_common_names, + Some(vec!["hive-agent-a".to_owned()]) + ); + assert_eq!(r.token_policies, Some(vec!["shared".to_owned()])); + assert_eq!(r.certificate, "PEM"); + } + #[test] fn a_complete_environment_is_accepted() { // The control: without this, every assertion below could be passing