From 6170e74a31aabd9e2b5a39aa996e0a0719958de2 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 27 Sep 2026 19:30:38 +0200 Subject: [PATCH] swarm-bao: agent certificates issued by a store-generated agent CA An agent's store identity was signed in swarm-controller's memory by a CA a controller-host unit generated on disk, and the listener never trusted that CA. Agent leaves now come from the store itself: a `pki-agents` PKI mount whose root openbao generates internally, so the agent CA's key never exists outside the store. - swarm-bao-agent-pki (new, store host, as the bao granter): enables and tunes the mount, generates the root once (guarded on an empty issuer list, no replace branch), upserts the `swarm-agent` role (client certificates named `hive-agent-*` only, 90 days), caches the CA at /var/lib/swarm-bao-tls/agent-ca.pem and composes the listener bundle. - The listener's tls_client_ca_file is a new listener-client-ca.pem (client-ca.pem, then the agent CA). Host cert-auth roles still pin client-ca.pem, so an agent leaf satisfies no host role. swarm-bao-certs composes the same bundle before openbao starts. - openbao reads tls_client_ca_file only at start, so when the bundle changed after openbao started, swarm-bao-agent-pki restarts openbao.service in the container; under `seal = "shamir"` it prints the step instead. Once swarm-bao-certs has a cached CA, later boots start openbao with it and do not restart. - The controller policy gains exactly `update` on pki-agents/issue/swarm-agent. mint_and_verify now asks that role for the leaf (the store generates the key), writes the agent's cert-auth role pinning the issuing CA bao returned, and writes the agent's policy as render_agent alone: the hive-shared queue credential stanza is gone. - deploy.bao.agentPkiRoleName (must start `swarm-`, asserted with the other pki role names); swarm-controller gets SWARM_CONTROLLER_AGENT_PKI_MOUNT/_ROLE from the deploy.bao options. Deleted: swarm-controller-agent-ca and its options (agentCaFile, agentCaKeyFile), env, LoadCredential entries and assertion; agent_identity's Authority, rcgen signing and validity window; the rcgen and time dependencies of swarm-controller (rcgen leaves the workspace); policy::render_agent_with_queue and its tests. The CN-prefix assertion policy.rs said was owed is not: agent and host roles pin different CAs. Migration is re-creating each agent after deploy; that overwrites the stale role and policy. Closes #4756 --- Cargo.lock | 203 +---------- Cargo.toml | 14 - docs/swarm/credentials.md | 26 +- docs/swarm/secrets.md | 12 +- nix/host-modules/swarm-bao.nix | 314 +++++++++++++++-- nix/host-modules/swarm-controller.nix | 152 +-------- nix/module-eval/bao-basics.nix | 67 ++++ nix/module-eval/bao-controller.nix | 50 +++ nix/module-eval/bao-grants.nix | 113 +++++- swarm-controller/Cargo.toml | 9 - swarm-controller/src/agent_identity.rs | 453 ++++++++----------------- swarm-controller/src/main.rs | 87 ++--- swarm-secret-client/src/client.rs | 150 +++++++- swarm-secret-client/src/forge.rs | 6 +- swarm-secret-client/src/lib.rs | 5 + swarm-secret-client/src/policy.rs | 162 ++------- 16 files changed, 896 insertions(+), 927 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 35c9cc36..11a6dd49 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -162,45 +162,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9dbc3a507a82b17ba0d98f6ce8fd6954ea0c8152e98009d36a40d8dcc8ce078a" -[[package]] -name = "asn1-rs" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" -dependencies = [ - "asn1-rs-derive", - "asn1-rs-impl", - "displaydoc", - "nom", - "num-traits", - "rusticata-macros", - "thiserror 2.0.18", - "time", -] - -[[package]] -name = "asn1-rs-derive" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", - "synstructure 0.13.2", -] - -[[package]] -name = "asn1-rs-impl" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "assign" version = "1.1.1" @@ -237,7 +198,7 @@ version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d83a251fa1a4c9d0fe6e816b7acd60549e473e08d14f27a1d992c2675abff05f" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-util", "memchr", @@ -347,7 +308,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ "axum-core", - "base64 0.22.1", + "base64", "bytes", "form_urlencoded", "futures-util", @@ -412,12 +373,6 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "base64" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" - [[package]] name = "base64ct" version = "1.8.3" @@ -430,22 +385,13 @@ version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f3c067aa24dd4ed5c79cf222a38f260c8f23d3b82a062fba3f28c6fe563b753" dependencies = [ - "base64 0.22.1", + "base64", "blowfish", "getrandom 0.4.3", "subtle", "zeroize", ] -[[package]] -name = "bit-vec" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" -dependencies = [ - "serde", -] - [[package]] name = "bitflags" version = "2.13.1" @@ -1089,20 +1035,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "der-parser" -version = "10.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" -dependencies = [ - "asn1-rs", - "displaydoc", - "nom", - "num-bigint", - "num-traits", - "rusticata-macros", -] - [[package]] name = "deranged" version = "0.5.8" @@ -1808,7 +1740,7 @@ dependencies = [ "anyhow", "async-nats", "axum", - "base64 0.22.1", + "base64", "bcrypt", "chrono", "clap", @@ -2269,7 +2201,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-channel", "futures-util", @@ -2982,7 +2914,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fef37395fffb7c916f7109ab0d16d8ca599403dd8164d08c0d966176b66ede47" dependencies = [ "async-trait", - "base64 0.22.1", + "base64", "futures-util", "getrandom 0.4.3", "gloo-utils", @@ -3041,7 +2973,7 @@ version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f48f304e553fb6200b1d7d1f77a88fd182076d4b25624e9dbfa42d6a37de35e" dependencies = [ - "base64 0.22.1", + "base64", "blake3", "chacha20poly1305", "getrandom 0.2.17", @@ -3124,12 +3056,6 @@ dependencies = [ "unicase", ] -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - [[package]] name = "miniz_oxide" version = "0.8.9" @@ -3185,16 +3111,6 @@ dependencies = [ "signatory", ] -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -3204,31 +3120,12 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "num-bigint" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" -dependencies = [ - "num-integer", - "num-traits", -] - [[package]] name = "num-conv" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" -[[package]] -name = "num-integer" -version = "0.1.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" -dependencies = [ - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -3254,7 +3151,7 @@ version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" dependencies = [ - "base64 0.22.1", + "base64", "chrono", "getrandom 0.2.17", "http", @@ -3277,15 +3174,6 @@ dependencies = [ "reqwest", ] -[[package]] -name = "oid-registry" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" -dependencies = [ - "asn1-rs", -] - [[package]] name = "once_cell" version = "1.21.4" @@ -3360,7 +3248,7 @@ version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" dependencies = [ - "base64 0.22.1", + "base64", "const-hex", "opentelemetry", "opentelemetry_sdk", @@ -3434,16 +3322,6 @@ dependencies = [ "digest 0.10.7", ] -[[package]] -name = "pem" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" -dependencies = [ - "base64 0.23.1", - "serde_core", -] - [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -3887,20 +3765,6 @@ dependencies = [ "rand_core 0.9.5", ] -[[package]] -name = "rcgen" -version = "0.14.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8" -dependencies = [ - "pem", - "ring", - "rustls-pki-types", - "time", - "x509-parser", - "yasna", -] - [[package]] name = "readlock" version = "0.1.11" @@ -3980,7 +3844,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "encoding_rs", "futures-channel", @@ -4142,7 +4006,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c3b4f00112791b490acce57df1ce3eb3f88899b045bebcff8a29f75369640cc" dependencies = [ "as_variant", - "base64 0.22.1", + "base64", "bytes", "date_header", "form_urlencoded", @@ -4259,15 +4123,6 @@ dependencies = [ "semver", ] -[[package]] -name = "rusticata-macros" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" -dependencies = [ - "nom", -] - [[package]] name = "rustify" version = "0.7.0" @@ -4890,7 +4745,7 @@ dependencies = [ "async-nats", "async-trait", "axum", - "base64 0.22.1", + "base64", "bytes", "forgejo-api", "futures-util", @@ -4907,7 +4762,6 @@ dependencies = [ "opentelemetry-otlp", "opentelemetry_sdk", "problem_details", - "rcgen", "reqwest", "serde", "serde_json", @@ -4917,7 +4771,6 @@ dependencies = [ "swarm-matrix-client", "swarm-queue-client", "swarm-secret-client", - "time", "tokio", "tracing", "url", @@ -5321,7 +5174,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f591660438b3038dd04d16c938271c79e7e06260ad2ea2885a4861bfb238605d" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-core", "futures-sink", @@ -5774,7 +5627,7 @@ checksum = "b98bf83c0992966775b8012f194b07b44928996163e5a05b741b43891571ae5b" dependencies = [ "aes", "arrayvec", - "base64 0.22.1", + "base64", "base64ct", "cbc", "chacha20poly1305", @@ -6264,40 +6117,12 @@ dependencies = [ "zeroize", ] -[[package]] -name = "x509-parser" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" -dependencies = [ - "asn1-rs", - "data-encoding", - "der-parser", - "lazy_static", - "nom", - "oid-registry", - "ring", - "rusticata-macros", - "thiserror 2.0.18", - "time", -] - [[package]] name = "xxhash-rust" version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "985eec839aaf2a1270af8f4ebcf63cf9401cfd90f0902f97c28d9f104ffbde72" -[[package]] -name = "yasna" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" -dependencies = [ - "bit-vec", - "time", -] - [[package]] name = "yoke" version = "0.8.3" diff --git a/Cargo.toml b/Cargo.toml index 2dd60d00..22eb6dda 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -112,20 +112,6 @@ vaultrs = "0.8" # resolves, since its `exec_with_empty` takes *its* `Endpoint` trait. rustify = "0.7" rustify_derive = "0.5" -# Signs the per-agent client leaf `swarm-controller::agent_identity` mints. -# A library rather than an `openssl` shellout, which is what every other CA -# in this tree is (`glue-bao-tls.nix`, `hive-tls.nix`, `swarm-ca.nix`): those -# run once at deploy time and write to disk, this one runs per agent -# creation and must keep the private key it generates in memory long enough -# to log in with it and no longer. `ring` over `aws_lc_rs` because `ring` is -# already in the lock; the crypto backend is not otherwise load-bearing. -# -# `x509-parser`: `Issuer::from_ca_cert_pem` is gated behind it, and reading -# the authority back out of its own PEM is how a leaf inherits the subject and -# key identifier that make it chain. The ungated constructors take a -# `CertificateParams` the caller would have to restate by hand — two spellings -# of one authority, agreeing until the day they don't. -rcgen = { version = "0.14", features = ["x509-parser"] } tower-http = { version = "0.7", features = ["fs"] } uuid = { version = "1", features = ["v4"] } rmcp = { version = "2", default-features = false, features = [ diff --git a/docs/swarm/credentials.md b/docs/swarm/credentials.md index 4be3c1e3..d816e502 100644 --- a/docs/swarm/credentials.md +++ b/docs/swarm/credentials.md @@ -54,19 +54,19 @@ strategy for every credential, including the mTLS leaf. -| store path | minter | reader — pulls at runtime, holds in memory | renewal | -| ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `swarm/agents//matrix/main` | `swarm-controller`, with the swarm's appservice token, at agent creation and in a five-minute pass | the agent container itself, under the certificate its hive passed in | the pass re-mints when the stored token is missing, unknown to the homeserver, or someone else's | -| `swarm/agents//matrix/` | `swarm-controller` | the agent container itself, under the certificate its hive passed in | must be stated | -| `swarm/controller/swarm-controller/matrix/appservice-token` | `swarm-matrix-ctl`, inside the `hive-matrix` container, once | `swarm-controller`, under its own certificate | none: the container keeps its copy and republishes it when the store's differs | -| `swarm/agents//bao-mtls` | `swarm-controller`, at agent creation | `hive-c0re`, under the hive's own certificate, when it writes the agent's container config | must be stated | -| `swarm/agents//queue` | `swarm-controller`, at agent creation | the agent container itself, under its own certificate — the identity it presents to the swarm queue, naming that one agent rather than its hive | none: the secret is fixed for the life of the agent and is revoked by deleting the path. A rotation mechanism is tracked as separate work, because rotating this credential needs a reconnect path — a queue client holding a revoked secret doesn't find out until it reconnects | -| `swarm/agents//forge-token` | `swarm-controller`, at agent creation and in a pass every 5 minutes over every agent with a store identity | the agent container itself, under its own certificate, fetched to `/run/hive-agent-forge-token/token` | the controller re-mints when the stored token is missing or no longer matches the forge (last eight characters and scopes); the agent re-fetches on a 10-minute timer | -| `swarm/hives//matrix/appservice-token` | one minter, on the authelia host | the hive process that presents the token to its homeserver, under the hive's own certificate | must be stated | -| `swarm/hives//matrix/sender-token` | `swarm-matrix-ctl`, in the `hive-matrix` container | `swarm-matrix-ctl` itself, under its own certificate, before it decides whether to mint, and hive-c0re's `stored_sender_token()`, under the hive's own certificate | must be stated | -| `swarm/hives//queue/agent` | authelia | the agent container presenting the OIDC client to the swarm queue, under its own certificate | must be stated | -| `swarm/services//oidc/client` | authelia | the service process that presents the client secret, under the certificate of the host it runs on | must be stated | -| _(not in the store)_ a hive's mTLS leaf | the store's own PKI, or an operator placing it by hand | its own client, off disk — the exception above, because it's what makes every other row's pull possible | must be stated | +| store path | minter | reader — pulls at runtime, holds in memory | renewal | +| ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `swarm/agents//matrix/main` | `swarm-controller`, with the swarm's appservice token, at agent creation and in a five-minute pass | the agent container itself, under the certificate its hive passed in | the pass re-mints when the stored token is missing, unknown to the homeserver, or someone else's | +| `swarm/agents//matrix/` | `swarm-controller` | the agent container itself, under the certificate its hive passed in | must be stated | +| `swarm/controller/swarm-controller/matrix/appservice-token` | `swarm-matrix-ctl`, inside the `hive-matrix` container, once | `swarm-controller`, under its own certificate | none: the container keeps its copy and republishes it when the store's differs | +| `swarm/agents//bao-mtls` | the store's agent PKI mount (`deploy.bao.agentPkiMountPath`), which generates the key, at `swarm-controller`'s request at agent creation | `hive-c0re`, under the hive's own certificate, when it writes the agent's container config | must be stated | +| `swarm/agents//queue` | `swarm-controller`, at agent creation | the agent container itself, under its own certificate — the identity it presents to the swarm queue, naming that one agent rather than its hive | none: the secret is fixed for the life of the agent and is revoked by deleting the path. A rotation mechanism is tracked as separate work, because rotating this credential needs a reconnect path — a queue client holding a revoked secret doesn't find out until it reconnects | +| `swarm/agents//forge-token` | `swarm-controller`, at agent creation and in a pass every 5 minutes over every agent with a store identity | the agent container itself, under its own certificate, fetched to `/run/hive-agent-forge-token/token` | the controller re-mints when the stored token is missing or no longer matches the forge (last eight characters and scopes); the agent re-fetches on a 10-minute timer | +| `swarm/hives//matrix/appservice-token` | one minter, on the authelia host | the hive process that presents the token to its homeserver, under the hive's own certificate | must be stated | +| `swarm/hives//matrix/sender-token` | `swarm-matrix-ctl`, in the `hive-matrix` container | `swarm-matrix-ctl` itself, under its own certificate, before it decides whether to mint, and hive-c0re's `stored_sender_token()`, under the hive's own certificate | must be stated | +| `swarm/hives//queue/agent` | authelia | `swarm-bao-queue-agent` on the hive's host, under its own per-hive certificate; no agent's policy reaches it | must be stated | +| `swarm/services//oidc/client` | authelia | the service process that presents the client secret, under the certificate of the host it runs on | must be stated | +| _(not in the store)_ a hive's mTLS leaf | the store's own PKI, or an operator placing it by hand | its own client, off disk — the exception above, because it's what makes every other row's pull possible | must be stated | diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index fac7ad76..11e1b1d0 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -325,6 +325,14 @@ plus one leaf per principal it runs (the table below names the options). Those a credentials that can't come out of the store, being what opens it; everything else a hive needs does. +Agents are the one principal whose leaf the store issues. Its `pki-agents` +mount (`deploy.bao.agentPkiMountPath`) holds an agent CA generated inside the +store, and `swarm-controller`, already logged in under its own host leaf, asks +that mount's one role for a `hive-agent-` client certificate at agent +creation. Host roles never pin that CA and agent roles pin only it, so an +agent's certificate opens that agent's own `swarm/agents//*` and +nothing else. + ### Per-principal identities Bao matches a cert-auth role on the certificate's subject, so a certificate is an @@ -414,7 +422,9 @@ hive domain behind the gateway, down. `network.exposeHostPorts` opens the port on the bridge firewall and nowhere else. Reaching the port grants nothing by itself: openbao answers nothing without a -client certificate signed by `deploy.bao.clientCaFile`. The passthrough carries +client certificate signed by `deploy.bao.clientCaFile` or by the store's own +agent CA. The listener reads both from `listener-client-ca.pem`, which no +cert-auth role pins. The passthrough carries whichever certificate the reader presents, unchanged. ## The constraint that decides where the root lives diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 98843109..988520b0 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -356,6 +356,9 @@ let # The swarm appservice token, read-only: the controller creates agents' # matrix accounts with it and never writes it. matrix-ctl mints and publishes # it (`matrixCtlPolicyText` below). + # + # The agent PKI grant is its only path on that mount: it can ask the one role + # for a certificate, not write that role or reach the issuer. controllerPolicyText = '' path "auth/cert/certs/hive-*" { capabilities = ["create", "update", "read", "delete"] @@ -380,6 +383,10 @@ let path "${credentialMountPath}/data/${swarmAppserviceTokenLeaf}" { capabilities = ["read"] } + + path "${agentPkiMountPath}/issue/${agentPkiRoleName}" { + capabilities = ["update"] + } ''; # The identity that copies authelia's minted OIDC client secrets into the @@ -489,7 +496,9 @@ let # # ⚠️ NOT bao's own client-auth PKI. That one is ./glue-bao-tls.nix's # self-signed CA under `/var/lib/swarm-bao-pki`, and it stays outside the - # store permanently: bao cannot issue the credential that opens bao. + # store permanently: bao cannot issue the credential a host opens bao with. + # Agent leaves come from `agentPkiMountPath`, requested by a principal that + # has already logged in. # # The mount's own root is generated into it by the bootstrap unit below and # its private key never leaves — `root/generate/internal` keeps it inside @@ -500,13 +509,6 @@ let # and has to spell it the same way. servicesPkiMountPath = baoDeploy.servicesPkiMountPath; - # The PKI mount agent client certificates are issued from. Its root is - # generated inside the store, so the agent CA's key never exists outside it. - # A mount of its own because the services mount holds exactly one issuer; a - # root apart from ./glue-bao-tls.nix's CA because that is what keeps an - # agent's certificate from satisfying any host role. - agentPkiMountPath = baoDeploy.agentPkiMountPath; - # Subject of the root generated into that mount. A label for a human reading # a chain, not an identity anything authenticates against — same fall-through # ./swarm-ca.nix:29-37 uses, and for the same reason: a hive that has set @@ -594,6 +596,77 @@ let } ''; + # The PKI mount agent client certificates are issued from. Its root is + # generated inside the store, so the agent CA's key never exists outside it. + # A mount of its own because the services mount holds exactly one issuer; a + # root apart from ./glue-bao-tls.nix's CA because that is what keeps an + # agent's certificate from satisfying any host role. + agentPkiMountPath = baoDeploy.agentPkiMountPath; + agentPkiRoleName = baoDeploy.agentPkiRoleName; + + # Every common name the agent role issues for: `swarm_secret_client`'s + # `policy::AGENT_PREFIX`, which names each agent's cert-auth role and + # policy too. Outside it the role refuses, so the controller cannot obtain + # a certificate for any other name. + agentCnGlob = "hive-agent-*"; + + # The anchor every agent's cert-auth role pins by value, so it is never + # replaced by a deploy. 262800h like `servicesPkiRootTtl`, and for the same + # reason: the mount is tuned to it before generation, or bao clamps it. + agentPkiRootTtl = "262800h"; + + # The agent leaf's window, pinned on the role. Nothing re-issues a leaf + # before it ends; an operator re-runs agent creation. + agentPkiLeafTtl = "2160h"; + + # The agent CA's certificate, cached on this host by `swarm-bao-agent-pki`, + # so `swarm-bao-certs` can compose the listener's bundle before the store + # is up. Public material. + agentCaCachePath = "${tlsDir}/agent-ca.pem"; + + # What the listener verifies client certificates against: `client-ca.pem` + # first, then the agent CA. A file of its own because every host cert-auth + # role pins `client-ca.pem`: with the agent CA in that file, every leaf the + # controller can request would satisfy every host role. + listenerClientCaPath = "${tlsDir}/listener-client-ca.pem"; + + # Writes `listenerClientCaPath` from `clientCaPath` and, when it parses, the + # cached agent CA. The file is replaced only when its bytes change, so its + # mtime says when the listener's trust last changed; `swarm-bao-agent-pki` + # restarts openbao on exactly that. Refuses, leaving the current file, when + # the result would not begin with `client-ca.pem`: host logins depend on it. + composeListenerBundle = '' + compose_listener_bundle() { + if [ ! -s ${clientCaPath} ]; then + echo "${clientCaPath} is missing or empty; not composing the listener bundle" >&2 + return 1 + fi + local tmp + tmp="$(mktemp -p ${tlsDir} .listener-client-ca.XXXXXX)" + cat ${clientCaPath} > "$tmp" + if [ -s ${agentCaCachePath} ]; then + if openssl x509 -noout -in ${agentCaCachePath} >/dev/null 2>&1; then + printf '\n' >> "$tmp" + cat ${agentCaCachePath} >> "$tmp" + else + echo "${agentCaCachePath} does not parse; the listener will not trust agent certificates" >&2 + fi + fi + if ! cmp -s -n "$(stat -c %s ${clientCaPath})" ${clientCaPath} "$tmp"; then + rm -f "$tmp" + echo "the composed listener bundle does not begin with ${clientCaPath}; leaving the current one" >&2 + return 1 + fi + if cmp -s "$tmp" ${listenerClientCaPath}; then + rm -f "$tmp" + else + chmod 0644 "$tmp" + mv -f "$tmp" ${listenerClientCaPath} + echo "wrote ${listenerClientCaPath}" + fi + } + ''; + # ── the four principals that used to share the hive's own leaf ───────────── # # 🩸 Each of the four reads exactly ONE path in the store, and until this @@ -760,7 +833,7 @@ let tls_key_file = serverKeyCredentialPath; } // lib.optionalAttrs (baoDeploy.clientCaFile != null) { - tls_client_ca_file = clientCaPath; + tls_client_ca_file = listenerClientCaPath; tls_require_and_verify_client_cert = true; }; @@ -1251,7 +1324,9 @@ in ⚠️ NOT the store's own client-auth PKI. That one is ./glue-bao-tls.nix's self-signed CA on disk, and it stays outside the - store permanently — bao cannot issue the credential that opens bao. + store permanently — bao cannot issue the credential a host opens bao + with. Agent certificates come from + {option}`services.hyperhive.deploy.bao.agentPkiMountPath`. ''; }; @@ -1269,6 +1344,17 @@ in ''; }; + agentPkiRoleName = lib.mkOption { + type = lib.types.str; + default = "swarm-agent"; + description = '' + Role on {option}`services.hyperhive.deploy.bao.agentPkiMountPath` + agent client certificates are issued through. It issues client + certificates named `hive-agent-*` and nothing else, and swarm-controller + may call its `issue` endpoint and no other path on the mount. + ''; + }; + servicesPkiRoleName = lib.mkOption { type = lib.types.str; default = "swarm-services"; @@ -1859,13 +1945,18 @@ in # else, so a role named otherwise is a 403 at deploy time. assertion = !haveGranter - || (lib.hasPrefix "swarm-" servicesPkiRoleName && lib.hasPrefix "swarm-" natsPkiRoleName); + || ( + lib.hasPrefix "swarm-" servicesPkiRoleName + && lib.hasPrefix "swarm-" natsPkiRoleName + && lib.hasPrefix "swarm-" agentPkiRoleName + ); message = '' services.hyperhive.deploy.bao.servicesPkiRoleName - (${servicesPkiRoleName}) and + (${servicesPkiRoleName}), services.hyperhive.deploy.bao.natsPkiRoleName (${natsPkiRoleName}) - must both start with `swarm-`: the bao granter that writes them may - write pki roles under that prefix only. + and services.hyperhive.deploy.bao.agentPkiRoleName + (${agentPkiRoleName}) must all start with `swarm-`: the bao granter + that writes them may write pki roles under that prefix only. ''; } ]; @@ -1929,6 +2020,7 @@ in "swarm-bao-forwarder-oidc-policy" "swarm-bao-services-issuer-policy" "swarm-bao-nats-tls-policy" + "swarm-bao-agent-pki" ]; # 🚫 No `swarm.otel.scrapeTargets.bao` entry any more, and its absence is @@ -2119,7 +2211,11 @@ in description = "deliver the swarm secret store's server certificate"; before = [ "container@${cfg.machine}.service" ]; requiredBy = [ "container@${cfg.machine}.service" ]; - path = [ pkgs.coreutils ]; + path = [ + pkgs.coreutils + pkgs.diffutils + pkgs.openssl + ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; @@ -2155,6 +2251,12 @@ in exit 1 fi install -m 0644 ${lib.escapeShellArg baoDeploy.clientCaFile} ${tlsDir}/client-ca.pem + + # Composed here as well as by `swarm-bao-agent-pki` so that openbao + # starts already trusting the cached agent CA, and so the file it + # refuses to start without exists before the store's first run. + ${composeListenerBundle} + compose_listener_bundle ''; }; @@ -2734,6 +2836,170 @@ in ''; }; + # The agent PKI mount: its root, its one role, the host's copy of the + # root's certificate, and the listener's trust in it. + # + # ⚠️ This unit RESTARTS openbao. openbao reads `tls_client_ca_file` only + # when a listener is built, at start: neither SIGHUP nor a reload re-reads + # it. So when the listener bundle changed after openbao last started, + # openbao is restarted here, which drops every client for the restart and + # the unseal. `swarm-bao-certs` composes the same bundle before every + # later start, so this happens when the agent CA first appears (or is + # replaced by a re-initialised store), not per boot. Under `shamir` a + # restart needs a human unseal, so there it prints the step instead. + # + # `after` the granting units so a restart does not cut their writes off. + systemd.services.swarm-bao-agent-pki = lib.mkIf haveGranter { + description = "set up the swarm agent PKI mount and make the store's listener trust it"; + after = [ + "container@${cfg.machine}.service" + "swarm-bao-controller-policy.service" + "swarm-bao-secret-publisher-policy.service" + "swarm-bao-matrix-ctl-policy.service" + "swarm-bao-matrix-token-policy.service" + "swarm-bao-queue-agent-policy.service" + "swarm-bao-grafana-oidc-policy.service" + "swarm-bao-otel-oidc-policy.service" + "swarm-bao-forwarder-oidc-policy.service" + "swarm-bao-services-issuer-policy.service" + "swarm-bao-nats-tls-policy.service" + ] + ++ granterAfter; + requires = [ "swarm-bao-pki.service" ]; + wantedBy = [ "multi-user.target" ]; + path = [ + baoCli + pkgs.coreutils + pkgs.diffutils + pkgs.openssl + ]; + environment = granterEnv; + # Same unseal wait as its siblings above. + startLimitBurst = 2880; + startLimitIntervalSec = 90000; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + Restart = "on-failure"; + RestartSec = 30; + }; + script = '' + set -euo pipefail + + ${granterLogin} + + # Asked rather than attempted: `secrets enable` errors on a path + # already in use. + mounts="$(bao secrets list -format=json)" + case "$mounts" in + *'"${agentPkiMountPath}/"'*) ;; + *) bao secrets enable -path=${agentPkiMountPath} pki ;; + esac + + # Before generation, on every run: an untuned mount silently clamps + # the root to 768h (see the services mount above). + bao secrets tune -max-lease-ttl=${agentPkiRootTtl} ${agentPkiMountPath} + + # Generated once, ever. Every agent's cert-auth role pins this root + # by value, so a second one locks every agent out; there is no + # replace branch, and the granter holds no delete on the root. The + # mount's own issuer list is the guard, for the reason the services + # root gives: `{}` is the only answer that means "none". + if issuers="$(bao list -format=json ${lib.escapeShellArg "${agentPkiMountPath}/issuers"})"; then + echo "the ${agentPkiMountPath} mount already has an issuer — leaving it alone" + elif [ "$issuers" = '{}' ]; then + echo "generating the swarm agent CA into the ${agentPkiMountPath} mount" + # `max_path_length=0`: this CA signs leaves only. + bao write -field=issuing_ca ${lib.escapeShellArg "${agentPkiMountPath}/root/generate/internal"} \ + common_name=${lib.escapeShellArg "swarm-bao-agent-ca ${servicesPkiRootLabel}"} \ + issuer_name=swarm-agent-ca \ + ttl=${agentPkiRootTtl} \ + max_path_length=0 \ + key_type=rsa \ + key_bits=4096 >/dev/null + else + echo "could not list the ${agentPkiMountPath} issuers — not generating a root over one that may exist" >&2 + exit 1 + fi + + # Upserted every run. The whole narrowing of what the controller can + # obtain: client certificates, named `hive-agent-*`, nothing else. + # `allow_localhost` and `server_flag` are on by default in bao, so + # they are turned off here, not left out. No `issuer_ref`: the mount + # holds one issuer, which is its default. + bao write ${lib.escapeShellArg "${agentPkiMountPath}/roles/${agentPkiRoleName}"} \ + allowed_domains=${lib.escapeShellArg agentCnGlob} \ + allow_glob_domains=true \ + allow_bare_domains=false \ + allow_subdomains=false \ + allow_wildcard_certificates=false \ + allow_localhost=false \ + allow_any_name=false \ + allow_ip_sans=false \ + enforce_hostnames=true \ + server_flag=false \ + client_flag=true \ + code_signing_flag=false \ + email_protection_flag=false \ + key_usage=DigitalSignature \ + key_type=ec \ + key_bits=256 \ + ttl=${agentPkiLeafTtl} \ + max_ttl=${agentPkiLeafTtl} + + ca="$(bao read -field=certificate ${lib.escapeShellArg "${agentPkiMountPath}/cert/ca"})" + if ! openssl x509 -noout <<<"$ca" >/dev/null 2>&1; then + echo "the ${agentPkiMountPath} CA that bao returned does not parse — not caching it" >&2 + exit 1 + fi + tmp="$(mktemp -p ${tlsDir} .agent-ca.XXXXXX)" + printf '%s\n' "$ca" > "$tmp" + if cmp -s "$tmp" ${agentCaCachePath}; then + rm -f "$tmp" + else + chmod 0644 "$tmp" + mv -f "$tmp" ${agentCaCachePath} + echo "wrote ${agentCaCachePath}" + fi + + ${composeListenerBundle} + compose_listener_bundle + + machine=${lib.escapeShellArg cfg.machine} + if ! systemctl --machine="$machine" is-active --quiet openbao.service; then + echo "openbao in $machine is not running; it reads ${listenerClientCaPath} when it starts" + exit 0 + fi + started="$(systemctl --machine="$machine" show --timestamp=us+utc -p ActiveEnterTimestamp --value openbao.service)" + started_us="$(date -u -d "$started" +%s%6N)" + written_us="$(stat -c %.6Y ${listenerClientCaPath} | tr -d .)" + if [ "$written_us" -le "$started_us" ]; then + echo "openbao started after ${listenerClientCaPath} last changed; nothing to pick up" + exit 0 + fi + + # Fail closed: host logins go through this file too. + if [ ! -s ${listenerClientCaPath} ] \ + || ! cmp -s -n "$(stat -c %s ${clientCaPath})" ${clientCaPath} ${listenerClientCaPath} \ + || ! openssl x509 -noout -in ${listenerClientCaPath} >/dev/null 2>&1; then + echo "${listenerClientCaPath} is empty, unparseable or does not begin with ${clientCaPath}; not restarting openbao" >&2 + exit 1 + fi + '' + + ( + if baoDeploy.seal == "pkcs11" then + '' + echo "${listenerClientCaPath} changed after openbao started; restarting openbao in $machine (it unseals itself)" + systemctl --machine="$machine" restart openbao.service + '' + else + '' + echo "${listenerClientCaPath} changed after openbao started. A restart seals a ${baoDeploy.seal} store, so it is left to you, as root on this host:" >&2 + echo " systemctl --machine=$machine restart openbao.service # then unseal" >&2 + '' + ); + }; + # The CA bind source is written at runtime by a host unit, so the # container has to start after it — otherwise nspawn sets up a mount # over a file that does not exist yet. @@ -2754,7 +3020,8 @@ in # /var, systemd owns `${stateDir}` through `StateDirectory=`, and # binding over it is what breaks the unit. bindMounts = { - # Read-only: `swarm-bao-certs` on the host is the only writer, and + # Read-only: host units write it (`swarm-bao-certs`, and + # `swarm-bao-agent-pki` for the agent CA and the listener bundle), and # the store has no reason to modify its own identity. ${tlsDir} = { hostPath = tlsDir; @@ -2985,12 +3252,15 @@ in ]; }; - # ⚠️ Upstream sets `restartIfChanged = false` on this unit, on - # purpose: a restart SEALS the store and disconnects every client. - # So a change to the settings above does NOT take effect on - # `nixos-rebuild switch` — it lands in the config file and waits. - # Restarting is an operator action with an unseal on the far side of - # it, which is why nothing here tries to be clever about it. + # ⚠️ Upstream sets `restartIfChanged = false` on this unit: a restart + # SEALS the store and disconnects every client. The container around + # it does restart on `nixos-rebuild switch` whenever its config + # (these settings included) changes: nixos-containers sets + # `restartTriggers` on `container@${cfg.machine}` and nothing here + # overrides its `restartIfChanged`. The only in-place restart of this + # unit is `swarm-bao-agent-pki` on the host, once, when the listener's + # client-CA bundle changed after openbao started, and only under the + # self-unsealing `pkcs11` seal. # This container's own journal forwarder, copied from an agent # container's (nix/agent-modules/otel.nix) because every container diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix index 336a017c..b89b47fb 100644 --- a/nix/host-modules/swarm-controller.nix +++ b/nix/host-modules/swarm-controller.nix @@ -38,33 +38,6 @@ let # file would be handed to a daemon that cannot use it. haveHiveClientCa = haveBaoIdentity && deployCfg.swarm-controller.hiveClientCaFile != null; - # The authority this daemon issues AGENT client leaves from — a different - # question from `hiveClientCaFile` above, which is the authority it *trusts* - # hives by. This one it signs with, so it needs the private key too. - # - # ⚠️ Deliberately NOT the store's own PKI (`glue-bao-tls.nix`'s - # `/var/lib/swarm-bao-pki`). A cert-auth role pins its authority by value, - # per role, so a role this daemon writes carries whatever authority this - # daemon hands it — which is what lets the controller mint from its own CA - # on its own host without anything being co-located and without any - # existing role changing. `swarm-controller/src/agent_identity.rs`'s module - # doc is the long form. - agentCaDir = "/var/lib/swarm-controller-agent-ca"; - - # No authority named means mint one here. The alternative — leaving agent - # identities off until an operator places a CA by hand — is the state where - # the whole path is configured and silently does nothing, which is the - # failure mode `glue-bao-tls.nix` avoids the same way. - selfSignAgentCa = deployCfg.swarm-controller.agentCaFile == null; - agentCaCert = - if selfSignAgentCa then "${agentCaDir}/ca.pem" else deployCfg.swarm-controller.agentCaFile; - agentCaKey = - if selfSignAgentCa then "${agentCaDir}/ca-key.pem" else deployCfg.swarm-controller.agentCaKeyFile; - - # Minting an agent's identity means publishing it to the store, so the - # authority alone is not enough — same rule `haveHiveClientCa` states. - haveAgentCa = haveBaoIdentity && agentCaKey != null; - # `swarm_secret_client` reads these spellings explicitly rather than # vaultrs's `VAULT_*` defaults — falling through to those builds a client # with no identity and fails at the TLS handshake, naming neither. `%d` and @@ -88,12 +61,11 @@ let # to put in each hive's cert-auth role. SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE = "%d/hive-client-ca.pem"; } - // lib.optionalAttrs haveAgentCa { - # The authority agent leaves are ISSUED FROM, so unlike every other - # `*_CA_FILE` here it comes with a key. `%d` for both: the key is - # `0600` and root-owned, and this daemon runs unprivileged. - SWARM_CONTROLLER_AGENT_CA_FILE = "%d/agent-ca.pem"; - SWARM_CONTROLLER_AGENT_CA_KEY_FILE = "%d/agent-ca-key.pem"; + // lib.optionalAttrs haveBaoIdentity { + # Where agent leaves are issued. The store host sets that mount and + # role up from the same two options, which keeps the spellings equal. + SWARM_CONTROLLER_AGENT_PKI_MOUNT = deployCfg.bao.agentPkiMountPath; + SWARM_CONTROLLER_AGENT_PKI_ROLE = deployCfg.bao.agentPkiRoleName; }; # What `swarmctl` needs in order to act on authelia from the host. @@ -663,47 +635,6 @@ in ''; }; - agentCaFile = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = null; - example = "/var/lib/swarm-agent-ca/ca.pem"; - description = '' - Authority this daemon **issues** agent client certificates from, so - that an agent container can authenticate to the swarm secret store - under its own name. Read together with - {option}`services.hyperhive.deploy.swarm-controller.agentCaKeyFile`, - which is the private key it signs with. - - The mirror image of - {option}`services.hyperhive.deploy.swarm-controller.hiveClientCaFile`: - that one is an authority this daemon only *trusts by value*, so it is - public material and needs no key. This one signs, so it does. - - Leaving this `null` — the default — makes the module mint a - self-signed authority in `${agentCaDir}` on first boot and use that. - That is the ordinary shape: the store pins an authority per cert-auth - role, by value, so the authority agents are issued from does not have - to be the store's own PKI and does not have to live on the store's - host. Name a file here only when an operator issues agent leaves from - somewhere else; doing so turns the self-signing unit off. - ''; - }; - - agentCaKeyFile = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = null; - example = "/var/lib/swarm-agent-ca/ca-key.pem"; - description = '' - Private key for - {option}`services.hyperhive.deploy.swarm-controller.agentCaFile`. - Both or neither — an authority with no key signs nothing, and the - daemon refuses to start half-configured rather than looking ready. - - A path, never a value: the key's bytes in a nix expression land in - the world-readable nix store, permanently. - ''; - }; - queue = { clientSecretFile = lib.mkOption { type = lib.types.str; @@ -734,10 +665,7 @@ in services.hyperhive.swarm.otel.journaldUnits = [ "swarm-controller" "swarm-controller-credential" - ] - # Declared only where the unit exists — an entry for a unit that was - # never defined is a collector waiting on a journal that never speaks. - ++ lib.optional (haveAgentCa && selfSignAgentCa) "swarm-controller-agent-ca"; + ]; users.users.swarm-controller = { isSystemUser = true; @@ -824,18 +752,6 @@ in state directory. ''; } - { - assertion = - deployCfg.swarm-controller.agentCaFile == null || deployCfg.swarm-controller.agentCaKeyFile != null; - message = '' - services.hyperhive.deploy.swarm-controller.agentCaFile names an - authority but agentCaKeyFile is unset. - - The controller does not merely trust this authority, it issues - agent client certificates from it, so it needs the private key. - Set both, or set neither and let the module mint its own. - ''; - } ]; systemd.services.swarm-controller = { @@ -877,16 +793,7 @@ in ++ lib.optional ( haveBaoIdentity && deployCfg.bao.serverCaFile != null ) "bao-ca.pem:${deployCfg.bao.serverCaFile}" - ++ lib.optional haveHiveClientCa "hive-client-ca.pem:${deployCfg.swarm-controller.hiveClientCaFile}" - # The agent authority, key included — same shape and same reason as - # the store identity above: the key is root-owned `0600` and this - # daemon runs as `swarm-controller`. `swarm-controller-agent-ca` - # below is `requiredBy` this unit, so the files exist by the time - # systemd resolves these. - ++ lib.optionals haveAgentCa [ - "agent-ca.pem:${agentCaCert}" - "agent-ca-key.pem:${agentCaKey}" - ]; + ++ lib.optional haveHiveClientCa "hive-client-ca.pem:${deployCfg.swarm-controller.hiveClientCaFile}"; # The placeholder default that makes the above non-fatal. # `LoadCredential=` takes priority over `SetCredential=`, so this is @@ -1059,50 +966,5 @@ in ExecStart = "${pkgs.systemd}/bin/systemctl try-restart swarm-controller.service"; }; }; - - # The authority agent client leaves are issued from, minted here when the - # operator named none. Shape copied from ./glue-bao-tls.nix's - # `swarm-bao-pki`, including the rule that matters most: - # - # 🩸 Idempotent on ABSENCE, never on content. Re-issuing this CA would - # invalidate every agent leaf already published to the store AND every - # cert-auth role that pinned it by value, locking every agent container - # in the swarm out at once — on a rebuild that changed nothing an - # operator asked for. - # - # `before` + `requiredBy` rather than `after`: the daemon's - # `LoadCredential=` names these files by absolute path, and a - # `LoadCredential=` pointing at a file that is not there yet is fatal - # (`243/CREDENTIALS`), not a slow start. - systemd.services.swarm-controller-agent-ca = lib.mkIf (haveAgentCa && selfSignAgentCa) { - description = "mint the authority swarm agents' store certificates are issued from"; - before = [ "swarm-controller.service" ]; - requiredBy = [ "swarm-controller.service" ]; - path = [ - pkgs.openssl - pkgs.coreutils - ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - }; - script = '' - set -euo pipefail - install -d -m 0700 ${agentCaDir} - - if [ ! -s ${agentCaCert} ]; then - # `pathlen:0` — this authority signs leaves and nothing else. An - # intermediate under it would be a second issuer for the one name - # space the store matches agents by. - openssl req -x509 -newkey rsa:4096 -nodes -sha256 -days 3650 \ - -keyout ${agentCaKey} -out ${agentCaCert} \ - -subj "/CN=swarm-agent-ca ${swarmDomain}" \ - -addext "basicConstraints=critical,CA:TRUE,pathlen:0" \ - -addext "keyUsage=critical,keyCertSign,cRLSign" - chmod 0600 ${agentCaKey} - chmod 0644 ${agentCaCert} - fi - ''; - }; }; } diff --git a/nix/module-eval/bao-basics.nix b/nix/module-eval/bao-basics.nix index ad39ff7f..6f747faf 100644 --- a/nix/module-eval/bao-basics.nix +++ b/nix/module-eval/bao-basics.nix @@ -44,6 +44,8 @@ let # The store's units live inside its container, so the gates below have to # look there rather than at the host's service set. baoUnits = machine: machine.containers.swarm-bao.config.systemd.services; + + tlsDir = "/var/lib/swarm-bao-tls"; cases = [ { # The store's seal is spread over six gates — the stanza, the @@ -214,6 +216,71 @@ let name = "the store advertises a cluster address"; ok = lib.hasPrefix "https://" ((baoSettings baoPkcs11).cluster_addr or ""); } + { + # The listener trusts the agent CA through a file no cert-auth role + # names. At least one listener verifies clients, so an empty set cannot + # pass. + name = "every client-verifying listener reads the listener-only bundle"; + ok = + let + verifying = lib.filter (l: l ? tls_client_ca_file) ( + lib.attrValues (baoSettings baoPkcs11).listener + ); + in + verifying != [ ] + && lib.all (l: l.tls_client_ca_file == "${tlsDir}/listener-client-ca.pem") verifying; + } + { + # The other half of keeping agents out of host roles: those roles pin + # the store CA alone. The positive count is the control that the text + # searched is the one the roles are written in. + name = "host cert-auth roles pin client-ca.pem and never the listener bundle or the agent CA"; + ok = + let + scripts = lib.concatStrings ( + lib.mapAttrsToList ( + n: u: lib.optionalString (lib.hasPrefix "swarm-bao-" n) (u.script or "") + ) baoPkcs11.systemd.services + ); + in + lib.hasInfix "certificate=@${tlsDir}/client-ca.pem" scripts + && !(lib.hasInfix "certificate=@${tlsDir}/listener-client-ca.pem" scripts) + && !(lib.hasInfix "certificate=@${tlsDir}/agent-ca.pem" scripts); + } + { + # Both writers compose the bundle through one function, which refuses + # a result that does not begin with the store CA and replaces the file + # only when its bytes change. + name = "both bundle writers use the one composer, which keeps the store CA first"; + ok = + let + s = baoPkcs11.systemd.services; + composes = + u: + lib.hasInfix "compose_listener_bundle() {" u.script + && lib.hasInfix "\ncompose_listener_bundle\n" u.script + && lib.hasInfix ''cmp -s -n "$(stat -c %s ${tlsDir}/client-ca.pem)" ${tlsDir}/client-ca.pem "$tmp"'' u.script + && lib.hasInfix ''cmp -s "$tmp" ${tlsDir}/listener-client-ca.pem'' u.script + && lib.hasInfix "mktemp -p ${tlsDir} " u.script; + in + s ? swarm-bao-agent-pki && composes s.swarm-bao-certs && composes s.swarm-bao-agent-pki; + } + { + # openbao reads its client-CA file only at start, so picking up the + # agent CA is a restart: automatic where the store unseals itself, + # printed where a human has to. The shamir arm is the control. + name = "the agent PKI unit restarts openbao under pkcs11 and only prints the step under shamir"; + ok = + let + restart = ''systemctl --machine="$machine" restart openbao.service''; + p = baoPkcs11.systemd.services.swarm-bao-agent-pki.script; + sh = baoShamir.systemd.services.swarm-bao-agent-pki.script; + in + lib.hasInfix restart p + && lib.hasInfix ''if [ "$written_us" -le "$started_us" ]; then'' p + && !(lib.hasInfix restart sh) + && lib.hasInfix "# then unseal" sh; + } ]; in runGroup "bao-basics" cases diff --git a/nix/module-eval/bao-controller.nix b/nix/module-eval/bao-controller.nix index da8883e9..671ee747 100644 --- a/nix/module-eval/bao-controller.nix +++ b/nix/module-eval/bao-controller.nix @@ -31,6 +31,15 @@ let deploy.swarm-controller.enable = true; }; + # Store and controller with an agent PKI mount and role no default could + # supply. + controllerAgentPkiMarker = hive { + deploy.bao.enable = true; + deploy.bao.agentPkiMountPath = "pki-agents-marker"; + deploy.bao.agentPkiRoleName = "swarm-agent-marker"; + deploy.swarm-controller.enable = true; + }; + # The controller with no store, which is every spread deployment. Nothing # mints here, so the pairing must leave the paths unset rather than name # files this host will never have. @@ -220,6 +229,47 @@ let in lib.hasInfix "cn-marker-not-a-default" role && lib.hasInfix "cn-marker-not-a-default" pki; } + { + # Agent leaves come from the store's own agent PKI, so the controller + # holds no authority of its own: no minting unit, no key, no variable + # naming one. + name = "the controller holds no agent CA and is told the store's agent PKI mount and role"; + ok = + let + s = baoControllerHere.systemd.services; + e = s.swarm-controller.environment; + in + s ? swarm-controller + && !(s ? swarm-controller-agent-ca) + && !(e ? SWARM_CONTROLLER_AGENT_CA_FILE) + && !(e ? SWARM_CONTROLLER_AGENT_CA_KEY_FILE) + && !(lib.any (c: lib.hasPrefix "agent-ca" c) s.swarm-controller.serviceConfig.LoadCredential) + && (e.SWARM_CONTROLLER_AGENT_PKI_MOUNT or null) == "pki-agents" + && (e.SWARM_CONTROLLER_AGENT_PKI_ROLE or null) == "swarm-agent"; + } + { + # The controller issues through whatever mount and role it is told, and + # its grant names a path. Both read the store's options, which the + # marker values prove: no default could supply them. + name = "the controller's issue grant and its environment name one mount and role"; + ok = + let + s = controllerAgentPkiMarker.systemd.services; + in + s.swarm-controller.environment.SWARM_CONTROLLER_AGENT_PKI_MOUNT == "pki-agents-marker" + && s.swarm-controller.environment.SWARM_CONTROLLER_AGENT_PKI_ROLE == "swarm-agent-marker" + && lib.hasInfix ''path "pki-agents-marker/issue/swarm-agent-marker"'' s.swarm-bao-controller-policy.script; + } + { + # Absence arm: without a store identity nothing can be issued, so the + # variables are not set. + name = "a controller with no store leaf is not told an agent PKI"; + ok = + let + e = controllerNoStore.systemd.services.swarm-controller.environment; + in + !(e ? SWARM_CONTROLLER_AGENT_PKI_MOUNT) && !(e ? SWARM_CONTROLLER_AGENT_PKI_ROLE); + } ]; in runGroup "bao-controller" cases diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index b2ea4f39..4f175725 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -57,6 +57,12 @@ let deploy.bao.natsPkiRoleName = "queue"; }; + # An agent pki role the granter's `roles/swarm-*` does not reach. + baoGranterOddAgentRole = hive { + deploy.bao.enable = true; + deploy.bao.agentPkiRoleName = "agent"; + }; + # The store and the token, with no CA to trust. `mkForce` because the PKI # glue supplies one by default here — this is the deployment that brings its # own certificates and has not named the authority yet, in which nothing can @@ -145,7 +151,7 @@ let _: u: (u.environment.BAO_CLIENT_CERT or null) == granterCertFile ) baoGrantWithConsumers.systemd.services; - # The ten units that write a `swarm-*` grant, by name, for the discovery + # The eleven units that write a `swarm-*` grant, by name, for the discovery # control below. grantingUnitNames = [ "swarm-bao-controller-policy" @@ -158,6 +164,7 @@ let "swarm-bao-forwarder-oidc-policy" "swarm-bao-services-issuer-policy" "swarm-bao-nats-tls-policy" + "swarm-bao-agent-pki" ]; # Comment lines dropped first: both the HCL and the scripts explain @@ -741,24 +748,24 @@ let } { # A store host without the granter's pair writes its grants some other - # way, so none of the ten units may exist. Without this arm + # way, so none of the eleven units may exist. Without this arm # `lib.mkIf haveGranter` could be dropped from any of them and every other # case here would still pass. - name = "without the granter's pair none of the ten granting units render"; + name = "without the granter's pair none of the eleven granting units render"; ok = let s = baoGranterOptOut.systemd.services; in lib.all (unit: !(s ? ${unit})) (grantingUnitNames ++ [ "swarm-bao-granter-role" ]) - # The control: the same store with the pair renders all ten. + # The control: the same store with the pair renders all eleven. && lib.all (unit: baoGrantHere.systemd.services ? ${unit}) grantingUnitNames; } { - # 🩸 What replaced the silent skip. With no bootstrap token the ten still + # 🩸 What replaced the silent skip. With no bootstrap token the eleven still # render, and a refused granter fails them with the step that fixes it. # A store host that never named a token is told to name one, since the # unit that sets the granter up renders only where it has. - name = "a store host without a bootstrap token renders the ten, each failing loudly with the one-time step"; + name = "a store host without a bootstrap token renders the eleven, each failing loudly with the one-time step"; ok = let s = baoGranterNoToken.systemd.services; @@ -1122,8 +1129,8 @@ let } { # What makes the case above mean something: discovery by the granter's - # certificate reaches all ten units, and each yields calls. - name = "the granter-policy check sees all ten granting units, and parses calls from each"; + # certificate reaches all eleven units, and each yields calls. + name = "the granter-policy check sees all eleven granting units, and parses calls from each"; ok = lib.sort lib.lessThan (lib.attrNames granterUnits) == lib.sort lib.lessThan grantingUnitNames && lib.all (u: baoCalls u.script != [ ]) (lib.attrValues granterUnits) @@ -1163,6 +1170,96 @@ let ] && grantFor bootstrapGrants "sys/policies/acl/swarm-controller" == null; } + { + # The controller's whole reach on any PKI mount: one role's issue + # endpoint. It cannot rewrite the role, sign a CSR of its choosing or + # touch the issuer, so the role's narrowing is the narrowing. + name = "the controller's only PKI grant is update on the agent role's issue path"; + ok = + let + cg = grantsIn baoGrantHere.systemd.services.swarm-bao-controller-policy.script; + in + lib.filter (g: lib.hasInfix "pki" g.path) cg == [ + { + path = "pki-agents/issue/swarm-agent"; + caps = [ "update" ]; + } + ] + && lib.all (p: grantFor cg p == null) [ + "pki-agents/roles/swarm-agent" + "pki-agents/sign/swarm-agent" + "pki-agents/sign-verbatim/swarm-agent" + "pki-agents/issue/swarm-other" + "pki-agents/root/generate/internal" + "pki-agents/issuer/default" + "pki-agents/config/urls" + "pki-agents/keys" + "pki/issue/swarm-services" + "sys/mounts/pki-agents" + ]; + } + { + # The engine refuses any name outside the glob, which is what keeps a + # host CN out of the controller's reach. Exactly one unit writes a role + # on the agent mount, so no second role widens it. + name = "the agent PKI role issues client certificates named hive-agent-* and nothing else"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-agent-pki.script; + roleWriters = lib.filter (u: matches "bao write '?pki-agents/roles/" (u.script or "") != [ ]) ( + lib.attrValues baoGrantHere.systemd.services + ); + in + lib.all (t: lib.hasInfix t s) [ + "allowed_domains='hive-agent-*'" + "allow_glob_domains=true" + "allow_bare_domains=false" + "allow_subdomains=false" + "allow_wildcard_certificates=false" + "allow_localhost=false" + "allow_any_name=false" + "allow_ip_sans=false" + "server_flag=false" + "client_flag=true" + "code_signing_flag=false" + "email_protection_flag=false" + "ttl=2160h" + "max_ttl=2160h" + ] + && lib.length roleWriters == 1; + } + { + # Every agent's role pins this root by value: generated once, after the + # tune that stops bao clamping it, and never deleted. + name = "the agent root is generated once, after the tune, as a leaf-only CA, and nothing deletes it"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-agent-pki.script; + tune = "bao secrets tune -max-lease-ttl=262800h pki-agents"; + generate = "pki-agents/root/generate/internal"; + before = + a: b: + lib.stringLength (lib.head (lib.splitString b s)) + > lib.stringLength (lib.head (lib.splitString a s)); + in + lib.length (lib.splitString generate s) == 2 + && lib.hasInfix tune s + && before tune generate + && lib.hasInfix "max_path_length=0" s + && lib.hasInfix "elif [ \"$issuers\" = '{}' ]; then" s + && !(lib.hasInfix "bao delete" s) + && grantFor granterGrants "pki-agents/root" == null; + } + { + # The granter writes pki roles through `roles/swarm-*` only. + name = "an agent pki role name outside swarm-* is refused, naming the option"; + ok = + let + names = a: lib.hasInfix "services.hyperhive.deploy.bao.agentPkiRoleName" a.message; + in + lib.any (a: !a.assertion && names a) baoGranterOddAgentRole.assertions + && !(lib.any (a: !a.assertion && names a) baoGrantHere.assertions); + } ]; in runGroup "bao-grants" cases diff --git a/swarm-controller/Cargo.toml b/swarm-controller/Cargo.toml index 1e4dcf71..8ff62e1e 100644 --- a/swarm-controller/Cargo.toml +++ b/swarm-controller/Cargo.toml @@ -99,15 +99,6 @@ swarm-secret-client.workspace = true # The appservice calls `matrix_account::agent_token` mints agents' accounts # with — shared with `swarm-matrix-ctl`, which pins the same device id. swarm-matrix-client.workspace = true -# `agent_identity.rs` signs the per-agent client leaf the store authenticates -# an agent container by. The one runtime signer in this tree — every other CA -# here is a deploy-time `openssl` oneshot — because this one issues per agent -# creation and must keep the key it generates in memory, never on disk. -rcgen.workspace = true -# `rcgen::CertificateParams`'s validity window is a `time::OffsetDateTime`, -# which `agent_identity::validity` builds. Same workspace pin every other -# holder of a timestamp here uses. -time.workspace = true # `otel_http_client.rs`'s `AuthenticatedHttpClient` — an # `opentelemetry_http::HttpClient` impl authenticated with this crate's own # `swarm-queue-client` identity. Lives in this crate rather than diff --git a/swarm-controller/src/agent_identity.rs b/swarm-controller/src/agent_identity.rs index b3205f73..6b240973 100644 --- a/swarm-controller/src/agent_identity.rs +++ b/swarm-controller/src/agent_identity.rs @@ -1,14 +1,14 @@ -//! One agent's own identity at the swarm's secret store: minted here, +//! One agent's own identity at the swarm's secret store: issued by the store, //! published here, granted here — and, before the job node reports success, //! **used** here. //! -//! The swarm mints the agent's certificate so that no hive ever needs the -//! capability to mint one; the hive only carries it down. The controller is -//! the swarm-level service that does it because it already logs in to the -//! store, and its grant already covers exactly the objects written here -//! (`swarm-bao.nix`'s `controllerPolicyText`: `create/update` on -//! `secret/data/swarm/agents/*`, on `sys/policies/acl/hive-*`, and on -//! `auth/cert/certs/hive-*`). No new authority is asked for anywhere. +//! The swarm obtains the agent's certificate so that no hive ever needs the +//! capability to obtain one; the hive only carries it down. The controller is +//! the swarm-level service that asks because it already logs in to the store, +//! and its grant covers exactly the calls made here (`swarm-bao.nix`'s +//! `controllerPolicyText`: `update` on the agent PKI role's `issue` path, and +//! `create/update` on `secret/data/swarm/agents/*`, on +//! `sys/policies/acl/hive-*`, and on `auth/cert/certs/hive-*`). //! //! **Two credentials, deliberately unrelated.** The certificate reaches the //! store; the queue secret identifies the agent to the swarm queue. Both sit @@ -22,14 +22,11 @@ //! four — so [`mint_and_verify`] does not finish on a write. See //! [`read_back_as_agent`]. //! -//! ⚠️ The authority is **not** `/var/lib/swarm-bao-pki/ca-key.pem`. A -//! cert-auth role pins its authority by value, per role (see -//! [`SecretStore::write_cert_role`][swarm_secret_client::SecretStore::write_cert_role]), -//! so a role this daemon writes carries whatever authority this daemon hands -//! it — which is what lets the controller mint from its own CA on its own -//! host, with nothing co-located and no existing role changed. - -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +//! The authority is the store's own agent CA, generated inside its agent PKI +//! mount; its key never leaves the store. It signs no host leaf, and no host +//! role pins it, so an agent's certificate satisfies only that agent's role. +//! Nothing re-issues a leaf before it expires (the role's `ttl`); until a +//! renewal path exists, an operator re-runs agent creation. use anyhow::{Context, Result, bail}; use swarm_secret_client::{ @@ -37,163 +34,47 @@ use swarm_secret_client::{ client::{DEFAULT_CERT_MOUNT, Settings}, mtls, policy, queue, }; -use time::OffsetDateTime; -/// File holding the authority agent leaves are issued from, as -/// `swarm-controller.nix` names it. Public material. -pub const ENV_AGENT_CA: &str = "SWARM_CONTROLLER_AGENT_CA_FILE"; +/// The PKI mount agent leaves are issued from, as `swarm-controller.nix` sets +/// it from `deploy.bao.agentPkiMountPath`. +pub const ENV_AGENT_PKI_MOUNT: &str = "SWARM_CONTROLLER_AGENT_PKI_MOUNT"; -/// File holding the private key for [`ENV_AGENT_CA`]. 🩸 A path, never a -/// value — the key's bytes must not reach a unit file or the nix store. -pub const ENV_AGENT_CA_KEY: &str = "SWARM_CONTROLLER_AGENT_CA_KEY_FILE"; +/// The role on [`ENV_AGENT_PKI_MOUNT`] agent leaves are issued through, as +/// `swarm-controller.nix` sets it from `deploy.bao.agentPkiRoleName`. +pub const ENV_AGENT_PKI_ROLE: &str = "SWARM_CONTROLLER_AGENT_PKI_ROLE"; -/// How long a minted leaf is good for. -/// -/// Short enough that a leaked key is not permanent, long enough that the -/// absence of a renewal path is not immediately fatal. Nothing re-mints a leaf -/// today, so until a renewal path lands this is the interval after which an -/// operator re-runs agent creation. It is deliberately far shorter than the ten -/// years -/// `glue-bao-tls.nix` gives the store's own CA: that one is an authority -/// whose reissue invalidates every leaf under it, this one is a leaf. -/// Spelled in hours because `Duration::from_days` is not yet a stable `const -/// fn`; `read_policy`'s `RETRY_WINDOW` is the same workaround. -const LEAF_LIFETIME: Duration = Duration::from_hours(90 * 24); - -/// How far a leaf is backdated. -/// -/// The verifier is the store, on another machine: a certificate whose -/// `notBefore` is this exact instant is refused outright by a clock a second -/// behind ours, and the resulting error names a validity window rather than a -/// clock. -const CLOCK_SKEW: Duration = Duration::from_mins(5); - -/// The authority this daemon issues agent leaves from, loaded once at startup. -/// -/// ⚠️ **No `Debug` derive**, and the key field is private: this struct is -/// reachable from `WorkerDeps`, which is formatted nowhere today and is one -/// `#[derive(Debug)]` away from being formatted everywhere. -pub struct Authority { - /// The authority's certificate, PEM. Public material — it is also what - /// goes into each agent's cert-auth role and into each agent's published - /// credential. - ca_pem: String, - /// The authority's private key, PEM. Never logged, never published, - /// never leaves this struct. - key_pem: String, -} - -impl Authority { - /// Load the authority from the files [`ENV_AGENT_CA`] and - /// [`ENV_AGENT_CA_KEY`] name, or `None` when this host was given neither. - /// - /// `None` is a supported deployment, not a failure: it is the state every - /// controller is in before an operator has turned agent identities on, and - /// `run_swarm_node` reports it as that one node's named failure rather - /// than refusing to start the daemon. - /// - /// # Errors - /// When exactly one of the two variables is set — half an authority signs - /// nothing, and silently doing nothing about it is how a host ends up - /// looking configured — or when a named file cannot be read. - pub fn from_env() -> Result> { - match ( - std::env::var_os(ENV_AGENT_CA), - std::env::var_os(ENV_AGENT_CA_KEY), - ) { - (None, None) => Ok(None), - (Some(_), None) => bail!( - "{ENV_AGENT_CA} is set but {ENV_AGENT_CA_KEY} is not — an authority with no key signs nothing" - ), - (None, Some(_)) => bail!( - "{ENV_AGENT_CA_KEY} is set but {ENV_AGENT_CA} is not — a key with no certificate is not an authority" - ), - (Some(ca), Some(key)) => { - let ca_path = ca.to_string_lossy().into_owned(); - let key_path = key.to_string_lossy().into_owned(); - Ok(Some(Self { - ca_pem: std::fs::read_to_string(&ca_path).with_context(|| { - format!("reading the agent authority {ca_path} (from {ENV_AGENT_CA})") - })?, - key_pem: std::fs::read_to_string(&key_path).with_context(|| { - format!( - "reading the agent authority's key {key_path} (from {ENV_AGENT_CA_KEY})" - ) - })?, - })) - } - } - } - - /// Build one from PEM already in hand — the constructor a test uses, and - /// the one that keeps [`Authority::from_env`] the only place this process - /// reads a key off disk. - #[cfg(test)] - fn from_pem(ca_pem: String, key_pem: String) -> Self { - Self { ca_pem, key_pem } - } - - /// Issue a client leaf carrying `common_name`, returning `(certificate, - /// private key)` as PEM. - /// - /// `clientAuth` and nothing else: this certificate authenticates a - /// principal to the store and must not be usable to *serve* anything. - /// - /// # Errors - /// When the authority's own PEM will not parse, or the leaf will not sign. - fn mint_leaf(&self, common_name: &str) -> Result<(String, String)> { - let issuer_key = rcgen::KeyPair::from_pem(&self.key_pem) - .context("the agent authority's key is not a PEM key that can sign")?; - let issuer = rcgen::Issuer::from_ca_cert_pem(&self.ca_pem, issuer_key) - .context("the agent authority is not a PEM certificate that can issue")?; - - let (not_before, not_after) = validity(SystemTime::now(), LEAF_LIFETIME)?; - let mut params = rcgen::CertificateParams::default(); - params.distinguished_name = rcgen::DistinguishedName::new(); - params - .distinguished_name - .push(rcgen::DnType::CommonName, common_name); - params.is_ca = rcgen::IsCa::NoCa; - params.use_authority_key_identifier_extension = true; - params.key_usages = vec![ - rcgen::KeyUsagePurpose::DigitalSignature, - rcgen::KeyUsagePurpose::KeyEncipherment, - ]; - params.extended_key_usages = vec![rcgen::ExtendedKeyUsagePurpose::ClientAuth]; - params.not_before = not_before; - params.not_after = not_after; - - let leaf_key = rcgen::KeyPair::generate().context("generating the leaf's key")?; - let cert = params - .signed_by(&leaf_key, &issuer) - .with_context(|| format!("signing a leaf for {common_name}"))?; - Ok((cert.pem(), leaf_key.serialize_pem())) - } -} - -/// The validity window of a leaf minted at `now`, backdated by [`CLOCK_SKEW`]. -/// -/// A free function taking `now` rather than reading the clock itself, so the -/// arithmetic — the part that can be wrong by a factor of sixty — is testable -/// without waiting ninety days. +/// Where agent leaves are issued: `(mount, role)`, read from `get`. /// /// # Errors -/// When the system clock is before the unix epoch, or so far past it that the -/// window will not fit a timestamp. -fn validity(now: SystemTime, lifetime: Duration) -> Result<(OffsetDateTime, OffsetDateTime)> { - let secs = now - .duration_since(UNIX_EPOCH) - .context("the system clock is before the unix epoch")? - .as_secs(); - let secs = i64::try_from(secs).context("the system clock is past what a timestamp holds")?; - let skew = i64::try_from(CLOCK_SKEW.as_secs()).expect("a five-minute constant fits an i64"); - let life = i64::try_from(lifetime.as_secs()).context("the leaf lifetime does not fit")?; +/// Naming the first of the two variables that is unset or empty. +fn agent_pki(get: impl Fn(&str) -> Option) -> Result<(String, String)> { + let required = |var: &str| -> Result { + get(var) + .filter(|v| !v.is_empty()) + .with_context(|| format!("{var} is unset or empty, so no agent leaf can be issued")) + }; + Ok(( + required(ENV_AGENT_PKI_MOUNT)?, + required(ENV_AGENT_PKI_ROLE)?, + )) +} - let not_before = OffsetDateTime::from_unix_timestamp(secs - skew) - .context("the backdated start is not a representable time")?; - let not_after = OffsetDateTime::from_unix_timestamp(secs + life) - .context("the expiry is not a representable time")?; - Ok((not_before, not_after)) +/// What the cert-auth role for `agent` is written from. +struct RoleInputs<'a> { + /// Role name, policy name and the common name the role matches: one string. + name: String, + /// The authority the role pins: the one that signed this very leaf. + ca: &'a str, + /// The policy document the role attaches. + policy: String, +} + +fn role_inputs<'a>(agent: &str, credential: &'a mtls::Credential) -> Result> { + Ok(RoleInputs { + name: policy::agent_object_name(agent)?, + ca: &credential.ca, + policy: policy::render_agent(agent)?, + }) } /// How many bytes of kernel randomness a queue secret is before encoding. @@ -228,49 +109,46 @@ fn generate_queue_secret() -> Result { /// Give `agent` an identity at the store, and prove it works. /// -/// Five store writes' worth of agreement, then the login that checks it: +/// Five store calls' worth of agreement, then the login that checks it: /// -/// 1. mint a leaf whose common name is [`policy::agent_object_name`]; +/// 1. have the agent PKI role issue a leaf whose common name is +/// [`policy::agent_object_name`]; the store generates its key; /// 2. publish it at [`mtls::identity_path`], where the agent's hive collects /// it under the hive's own certificate; /// 3. publish a queue secret at [`queue::agent_queue_path`] — the agent's own /// identity at the swarm queue, minted here so that the credential an agent /// presents names *it* rather than its hive; -/// 4. write the ACL document [`policy::render_agent_with_queue`] renders — -/// read on this one agent's paths, plus the hive-shared queue credential -/// every agent container on `hive` already receives out of band; -/// 5. write the cert-auth role that ties the three together. +/// 4. write the ACL document [`policy::render_agent`] renders — read on this +/// one agent's paths and nothing else; +/// 5. write the cert-auth role that ties the three together, pinning the CA +/// the store named as this leaf's issuer. /// /// Policy before role, for the reason `read_policy::provision` gives: the role /// names the policy, so the other order leaves a window in which it points at /// nothing. /// -/// ⚠️ **Step 3 is idempotent and step 2 is not.** Re-running re-mints the -/// certificate — a fresh leaf the agent picks up on its next boot — but leaves -/// an existing queue secret alone. An agent holds that secret in a live -/// connection, and this function is re-run deliberately against agents that -/// are already running, so replacing it would drop them off the queue. -/// Nothing here rotates one; revoking means deleting the path. +/// ⚠️ **Step 3 is idempotent and steps 1–2 are not.** Re-running issues a +/// fresh leaf, picked up on the agent's next boot, but leaves an existing +/// queue secret alone: this is re-run against running agents, which hold that +/// secret in a live connection. Revoking one means deleting the path. /// /// # Errors /// Anything that stops one of those five steps, with the step named. A /// failure here fails the job node and nothing else — the agent is still -/// created, exactly as capable as every agent is today. -pub async fn mint_and_verify(authority: &Authority, agent: &str, hive: &str) -> Result<()> { +/// created, without a store identity. +pub async fn mint_and_verify(agent: &str, hive: &str) -> Result<()> { + let (mount, pki_role) = agent_pki(|k| std::env::var(k).ok())?; let name = policy::agent_object_name(agent)?; let path = mtls::identity_path(agent)?; let queue_path = queue::agent_queue_path(agent)?; - let (cert, key) = authority.mint_leaf(&name)?; - let credential = mtls::Credential { - cert, - key, - ca: authority.ca_pem.clone(), - }; - let store = crate::store::connect() .await .context("logging in to the swarm secret store")?; + let credential = store + .issue_client_certificate(&mount, &pki_role, &name) + .await + .with_context(|| format!("issuing {name}'s certificate from {mount}/issue/{pki_role}"))?; store .write(&path, &credential) .await @@ -319,32 +197,39 @@ pub async fn mint_and_verify(authority: &Authority, agent: &str, hive: &str) -> } let queue_credential = wanted; + let inputs = role_inputs(agent, &credential)?; store - .write_policy(&name, &policy::render_agent_with_queue(agent, hive)?) + .write_policy(&inputs.name, &inputs.policy) .await - .with_context(|| format!("writing the read policy {name}"))?; + .with_context(|| format!("writing the read policy {}", inputs.name))?; store - .write_cert_role(DEFAULT_CERT_MOUNT, &name, &authority.ca_pem, &name, &name) + .write_cert_role( + DEFAULT_CERT_MOUNT, + &inputs.name, + inputs.ca, + &inputs.name, + &inputs.name, + ) .await - .with_context(|| format!("writing the cert-auth role {name}"))?; + .with_context(|| format!("writing the cert-auth role {}", inputs.name))?; tracing::info!(agent, %path, role = %name, "agent store identity published"); read_back_as_agent(&credential, &name, &path, &queue_path, &queue_credential).await?; tracing::info!( agent, role = %name, - "agent store identity verified: the minted leaf logged in and read both its own paths" + "agent store identity verified: the issued leaf logged in and read both its own paths" ); Ok(()) } /// The consumer of everything [`mint_and_verify`] wrote: log in **as the -/// agent**, with the leaf just minted, and read back both paths just +/// agent**, with the leaf just issued, and read back both paths just /// published. /// /// The address and the store's CA come from this process's own `BAO_*` /// environment; the *identity* deliberately does not — see -/// [`SecretStore::connect_with_identity`]. The freshly minted private key +/// [`SecretStore::connect_with_identity`]. The freshly issued private key /// never touches a filesystem. /// /// Both paths, not just the certificate's, for the reason this function @@ -377,7 +262,7 @@ async fn read_back_as_agent( SecretStore::connect_with_identity(&settings, &identity, role, DEFAULT_CERT_MOUNT) .await .with_context(|| { - format!("logging in to the store as {role} with the leaf just minted") + format!("logging in to the store as {role} with the leaf just issued") })?; let read_back: mtls::Credential = as_agent .read(path) @@ -407,92 +292,10 @@ async fn read_back_as_agent( #[cfg(test)] mod tests { use super::{ - Authority, CLOCK_SKEW, LEAF_LIFETIME, QUEUE_SECRET_BYTES, generate_queue_secret, validity, + ENV_AGENT_PKI_MOUNT, ENV_AGENT_PKI_ROLE, QUEUE_SECRET_BYTES, agent_pki, + generate_queue_secret, role_inputs, }; - use std::time::{Duration, SystemTime, UNIX_EPOCH}; - - /// A throwaway CA, minted in-process so no test needs a fixture file. - fn test_authority() -> Authority { - let key = rcgen::KeyPair::generate().expect("a key generates"); - let mut params = rcgen::CertificateParams::default(); - params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Constrained(0)); - params - .distinguished_name - .push(rcgen::DnType::CommonName, "swarm-agent-ca"); - let ca = params.self_signed(&key).expect("the CA self-signs"); - Authority::from_pem(ca.pem(), key.serialize_pem()) - } - - #[test] - fn the_window_is_backdated_by_the_skew_and_as_long_as_the_lifetime() { - // The arithmetic that is wrong by a factor of sixty if a unit slips. - let now = UNIX_EPOCH + Duration::from_secs(1_700_000_000); - let (before, after) = validity(now, LEAF_LIFETIME).expect("a plain instant is fine"); - assert_eq!(before.unix_timestamp(), 1_700_000_000 - 300); - assert_eq!(after.unix_timestamp(), 1_700_000_000 + 90 * 24 * 60 * 60); - assert_eq!(CLOCK_SKEW, Duration::from_mins(5)); - } - - #[test] - fn a_minted_leaf_starts_valid_and_expires() { - let now = i64::try_from( - SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("the test host's clock is after 1970") - .as_secs(), - ) - .expect("and before the end of time"); - let (before, after) = validity(SystemTime::now(), LEAF_LIFETIME).expect("now is fine"); - assert!( - before.unix_timestamp() < now, - "a leaf usable only in the future is unusable" - ); - assert!( - after.unix_timestamp() > now, - "a leaf that has already expired authenticates nothing" - ); - // The rule `docs/swarm/credentials.md` states: no credential's - // renewal strategy may read NONE, which starts with it having an end. - // A decade-long leaf is that rule broken in a way review would miss. - assert!(after.unix_timestamp() - now < 3653 * 24 * 60 * 60); - } - - /// The four-strings agreement `mint_and_verify` rests on, checked on the - /// one of the four this process controls directly: the certificate really - /// does carry the common name the cert-auth role will be told to match. - #[test] - fn the_leaf_carries_the_agents_object_name_as_its_common_name() { - let name = swarm_secret_client::policy::agent_object_name("atlas").expect("legal"); - assert_eq!(name, "hive-agent-atlas"); - - let (cert, key) = test_authority().mint_leaf(&name).expect("the leaf signs"); - assert!(cert.contains("BEGIN CERTIFICATE"), "a PEM certificate"); - assert!(key.contains("PRIVATE KEY"), "a PEM key"); - - // Searched in the SIGNED DER rather than asserted on the params we - // built: the claim is that the name reached the bytes a verifier - // reads. A byte search rather than an X.509 parse because the whole - // crate would otherwise gain a parser dependency for one assertion — - // the name is a UTF8String in the subject DN, so it appears verbatim. - let body: String = cert - .lines() - .filter(|l| !l.starts_with("-----")) - .collect::>() - .join(""); - let der = base64::Engine::decode(&base64::engine::general_purpose::STANDARD, body) - .expect("the PEM body is base64"); - assert!( - der.windows(name.len()).any(|w| w == name.as_bytes()), - "the common name must be inside the signed certificate" - ); - - // And the pair is a usable client identity — the first thing - // `read_back_as_agent` does with it, in exactly this shape. - let mut identity = cert.into_bytes(); - identity.push(b'\n'); - identity.extend_from_slice(key.as_bytes()); - reqwest::Identity::from_pem(&identity).expect("the leaf and its key form a TLS identity"); - } + use swarm_secret_client::{mtls, policy}; /// The alphabet claim the token format rests on: the secret is carried in /// a composite the verifying end splits on `.`, so a secret that could @@ -516,43 +319,65 @@ mod tests { assert!(!a.contains('='), "{a}"); } - /// A misconfiguration that would otherwise look like "not configured": - /// half an authority has to be an error, not a silent `None`. - /// - /// SAFETY: single-threaded mutation of two env vars no other test in this - /// crate reads, removed again before returning. + fn both(k: &str) -> Option { + match k { + ENV_AGENT_PKI_MOUNT => Some("pki-agents".to_owned()), + ENV_AGENT_PKI_ROLE => Some("swarm-agent".to_owned()), + _ => None, + } + } + #[test] - fn half_an_authority_is_an_error_and_neither_half_is_absence() { - unsafe { - std::env::remove_var(super::ENV_AGENT_CA); - std::env::remove_var(super::ENV_AGENT_CA_KEY); - } - assert!( - Authority::from_env() - .expect("neither set is a supported shape") - .is_none(), - "a controller with no authority configured is not an error" + fn the_issue_path_comes_from_the_two_variables_the_module_sets() { + assert_eq!( + agent_pki(both).expect("both set"), + ("pki-agents".to_owned(), "swarm-agent".to_owned()) ); + } - unsafe { std::env::set_var(super::ENV_AGENT_CA, "/nonexistent/ca.pem") } - // `.err().expect(..)` rather than `expect_err`: the `Ok` half is an - // `Authority`, which deliberately has no `Debug` (it holds a key). - let e = Authority::from_env() - .err() - .expect("a certificate with no key is half an authority"); - assert!(format!("{e:#}").contains(super::ENV_AGENT_CA_KEY), "{e:#}"); - - unsafe { - std::env::remove_var(super::ENV_AGENT_CA); - std::env::set_var(super::ENV_AGENT_CA_KEY, "/nonexistent/ca-key.pem"); + #[test] + fn a_missing_or_empty_pki_variable_is_named() { + for var in [ENV_AGENT_PKI_MOUNT, ENV_AGENT_PKI_ROLE] { + let unset = agent_pki(|k| if k == var { None } else { both(k) }) + .expect_err("one variable is unset"); + assert!(format!("{unset:#}").contains(var), "{unset:#}"); + let empty = agent_pki(|k| { + if k == var { + Some(String::new()) + } else { + both(k) + } + }) + .expect_err("one variable is empty"); + assert!(format!("{empty:#}").contains(var), "{empty:#}"); } - // `.err().expect(..)` rather than `expect_err`: the `Ok` half is an - // `Authority`, which deliberately has no `Debug` (it holds a key). - let e = Authority::from_env() - .err() - .expect("a key with no certificate is the other half"); - assert!(format!("{e:#}").contains(super::ENV_AGENT_CA), "{e:#}"); + } - unsafe { std::env::remove_var(super::ENV_AGENT_CA_KEY) } + /// Three of the four strings that must agree, checked where this process + /// sets them: role, policy and matched CN are one name; the pinned CA is + /// the issuer the store reported for this leaf; the policy is the agent's + /// own single stanza. + #[test] + fn the_role_pins_the_leafs_own_issuer_under_the_agents_name() { + let credential = mtls::Credential { + cert: "LEAF".to_owned(), + key: "KEY".to_owned(), + ca: "AGENT-CA".to_owned(), + }; + let inputs = role_inputs("atlas", &credential).expect("legal"); + assert_eq!(inputs.name, "hive-agent-atlas"); + assert_eq!( + inputs.name, + policy::agent_object_name("atlas").expect("legal"), + "the CN the leaf is issued for" + ); + assert_eq!(inputs.ca, "AGENT-CA"); + assert_eq!(inputs.policy, policy::render_agent("atlas").expect("legal")); + assert!(!inputs.policy.contains("swarm/hives/"), "{}", inputs.policy); + + // The control: another agent's inputs differ in both name and grant. + let other = role_inputs("argus", &credential).expect("legal"); + assert_ne!(other.name, inputs.name); + assert!(!other.policy.contains("agents/atlas/"), "{}", other.policy); } } diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index b7edb801..d20d50df 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -101,9 +101,9 @@ enum SwarmNodeKind { /// report success on a write. /// /// Carries the hive for a different reason than `TriggerDeploy` does: - /// not as an address, but because an agent's ACL document grants read on - /// its hive's shared queue credential, so the document cannot be rendered - /// without knowing which hive the agent belongs to. + /// not as an address, but because the agent's queue credential names the + /// hive it may take subjects on, so it cannot be written without knowing + /// which hive the agent belongs to. MintAgentIdentity { hive: String, agent: String }, /// Make sure `agent` holds a live forge access token in the swarm secret /// store, minting one with the forge's admin API when it does not. See @@ -209,11 +209,6 @@ struct WorkerDeps { /// connection living there is an accident of construction order, not a /// claim that events are a kind of status. queue: Option, - /// The authority agent client leaves are issued from, loaded once at - /// startup because it holds a private key and a per-node re-read would be - /// a per-node chance to read one. `None` on a host the operator has not - /// given an authority — see `agent_identity::Authority::from_env`. - agent_ca: Option>, /// The wanted-state writer, for nodes that declare what a hive should /// converge an agent to. Shares the status reader's queue connection — /// see `wanted_writer`. `None` exactly when no swarm queue is configured @@ -319,9 +314,7 @@ async fn run_swarm_node( Err(e) => Outcome::Failed(format!("{e:#}")), }, }, - SwarmNodeKind::MintAgentIdentity { hive, agent } => { - mint_identity(deps.agent_ca.as_deref(), &agent, &hive).await - } + SwarmNodeKind::MintAgentIdentity { hive, agent } => mint_identity(&agent, &hive).await, SwarmNodeKind::MintAgentForgeToken { agent } => mint_forge_token(deps.forge, &agent).await, SwarmNodeKind::MintAgentMatrixAccount { agent } => { mint_matrix_account(deps.matrix_homeserver.as_deref(), &agent).await @@ -347,22 +340,10 @@ async fn run_swarm_node( /// The `MintAgentIdentity` arm, lifted out so `run_swarm_node` stays under /// `clippy::too_many_lines`. -async fn mint_identity( - authority: Option<&agent_identity::Authority>, - agent: &str, - hive: &str, -) -> hive_jobq::scheduler::Outcome { +async fn mint_identity(agent: &str, hive: &str) -> hive_jobq::scheduler::Outcome { use hive_jobq::scheduler::Outcome; - let Some(authority) = authority else { - return Outcome::Failed( - "no agent certificate authority is configured on this host \ - (SWARM_CONTROLLER_AGENT_CA_FILE / SWARM_CONTROLLER_AGENT_CA_KEY_FILE unset), \ - so this agent has no identity at the swarm secret store" - .to_owned(), - ); - }; - match agent_identity::mint_and_verify(authority, agent, hive).await { + match agent_identity::mint_and_verify(agent, hive).await { Ok(()) => Outcome::Done, Err(e) => Outcome::Failed(format!("{e:#}")), } @@ -1483,28 +1464,6 @@ fn name_verdict( } } -/// The authority agent leaves are issued from, or `None` on a host that was -/// given none. -/// -/// Same "log and carry on" shape as `main`'s other optional wiring: a -/// controller with no agent authority still serves everything else, and -/// `MintAgentIdentity` fails with a named reason rather than this process -/// refusing to start. The `Err` arm is worth its own warning — half an -/// authority, or a file that will not read, is a host that looks configured -/// and mints nothing. -fn load_agent_authority() -> Option> { - match agent_identity::Authority::from_env() { - Ok(authority) => authority.map(Arc::new), - Err(e) => { - tracing::warn!( - error = %format!("{e:#}"), - "agent certificate authority unusable; agents get no store identity here" - ); - None - } - } -} - /// The sub-DAG one agent creation is: the nodes, and the edges between them. /// /// A function rather than a closure inside [`create_agent`] so the endpoint's @@ -1589,9 +1548,9 @@ fn declare_agent_job( // // `after_any` on the mint, not `after_ok`: a hive cannot pass down a // certificate the swarm has not published, so the deploy must not - // overtake the mint — but a host with no authority configured must still - // create agents exactly as it does today. `after_ok` there would turn an - // unconfigured option into an agent nobody runs. + // overtake the mint — but a host whose store or agent PKI is not set up + // must still create agents. `after_ok` there would turn a store outage + // into an agent nobody runs. // // The forge-token mint gets `after_any` for the same reason: a host with // no forge or no store must still create agents; only that node fails, @@ -2300,7 +2259,6 @@ async fn main() -> Result<()> { auth: auth.clone(), forge: forge_client.clone(), queue: status.as_ref().map(|s| s.queue_client()), - agent_ca: load_agent_authority(), wanted: wanted_writer(status.as_ref()), matrix_homeserver: configured_matrix_homeserver(), }; @@ -2972,7 +2930,6 @@ mod tests { auth: None, forge: None, queue: None, - agent_ca: None, wanted: None, matrix_homeserver: None, }; @@ -3027,7 +2984,6 @@ mod tests { auth: None, forge: None, queue: None, - agent_ca: None, wanted: None, matrix_homeserver: None, }; @@ -3052,16 +3008,16 @@ mod tests { } /// Third sibling of the two above, and the same deliberate caveat: with - /// no authority configured this reaches only the - /// graceful-absence-is-failure branch. The happy path is a live store - /// and a real login, which is exactly why it is `mint_and_verify`'s own - /// job to prove it at agent-creation time rather than a unit test's. + /// no agent PKI named this reaches only the graceful-absence-is-failure + /// branch, before any network call. The happy path is a live store and a + /// real login, which is exactly why it is `mint_and_verify`'s own job to + /// prove it at agent-creation time rather than a unit test's. /// - /// What this does pin is the degrade: a host that was never given an - /// authority fails this one node with a reason that names the two - /// variables, and creates the agent anyway. + /// What this does pin is the degrade: a host whose environment does not + /// name the agent PKI fails this one node with a reason that names the + /// variable, and creates the agent anyway. #[tokio::test] - async fn mint_agent_identity_node_runs_end_to_end_and_fails_without_an_authority() { + async fn mint_agent_identity_node_runs_end_to_end_and_fails_without_the_agent_pki_named() { let mut sched = hive_jobq::scheduler::Scheduler::new( hive_jobq::Graph::new(), hive_jobq::resources::ResourceTable::new(), @@ -3082,7 +3038,6 @@ mod tests { auth: None, forge: None, queue: None, - agent_ca: None, wanted: None, matrix_homeserver: None, }; @@ -3101,9 +3056,8 @@ mod tests { assert_eq!(node.state, hive_jobq::State::Failed); let error = node.error.as_deref().unwrap_or_default(); assert!( - error.contains(crate::agent_identity::ENV_AGENT_CA) - && error.contains(crate::agent_identity::ENV_AGENT_CA_KEY), - "the reason must name both variables an operator has to set, got {error:?}" + error.contains(crate::agent_identity::ENV_AGENT_PKI_MOUNT), + "the reason must name the variable an operator has to set, got {error:?}" ); } @@ -3135,7 +3089,6 @@ mod tests { auth: None, forge: None, queue: None, - agent_ca: None, wanted: None, matrix_homeserver: None, }; @@ -3304,7 +3257,7 @@ mod tests { .expect("the deploy waits for the mint"); assert!( when.accepts(hive_jobq::TerminalState::Failed), - "an unconfigured authority must not cancel the deploy; this edge \ + "a failed or unconfigured agent PKI issue must not cancel the deploy; this edge \ has to be `after_any`, not `after_ok`" ); } diff --git a/swarm-secret-client/src/client.rs b/swarm-secret-client/src/client.rs index a0df7d36..4b5d8e05 100644 --- a/swarm-secret-client/src/client.rs +++ b/swarm-secret-client/src/client.rs @@ -2,9 +2,12 @@ use rustify_derive::Endpoint; use serde::{Serialize, de::DeserializeOwned}; -use vaultrs::client::{Client, VaultClient, VaultClientSettingsBuilder}; +use vaultrs::{ + api::pki::{requests::GenerateCertificateRequest, responses::GenerateCertificateResponse}, + client::{Client, VaultClient, VaultClientSettingsBuilder}, +}; -use crate::{Error, path::MOUNT}; +use crate::{Error, mtls, path::MOUNT}; /// The store's address. pub const ENV_ADDR: &str = "BAO_ADDR"; @@ -261,6 +264,30 @@ impl SecretStore { Ok(()) } + /// Have the PKI role `role` on `mount` issue a client certificate for + /// `common_name`, returning it with its key and the issuing CA. + /// + /// The store generates the key: it exists in the store's answer and in the + /// returned value, nowhere else. What the certificate may carry (names, + /// usages, lifetime) is the role's to decide, so nothing but the name is + /// asked for here. + /// + /// # Errors + /// [`Error::Vault`] when the token's policy does not cover + /// `/issue/` or the role refuses `common_name`, and + /// [`Error::IncompleteIssue`] when the answer lacks any of the three. + pub async fn issue_client_certificate( + &self, + mount: &str, + role: &str, + common_name: &str, + ) -> Result { + let issued = + vaultrs::api::exec_with_result(&self.inner, issue_request(mount, role, common_name)) + .await?; + credential_from_issue(issued) + } + /// The names of every cert-auth role under `mount`, or none when the /// mount has no roles at all. /// @@ -296,6 +323,45 @@ struct WriteAclPolicy { policy: String, } +/// The request [`SecretStore::issue_client_certificate`] sends. +/// +/// The name stays out of the SANs so the certificate carries exactly one name, +/// the one the cert-auth role matches. PKCS#8 because that is the key shape +/// every reader of the published identity already parses. +fn issue_request(mount: &str, role: &str, common_name: &str) -> GenerateCertificateRequest { + GenerateCertificateRequest { + mount: mount.to_owned(), + role: role.to_owned(), + common_name: Some(common_name.to_owned()), + exclude_cn_from_sans: Some(true), + private_key_format: Some("pkcs8".to_owned()), + ..GenerateCertificateRequest::default() + } +} + +/// The store's answer, moved straight into the redacting +/// [`mtls::Credential`]. `GenerateCertificateResponse` derives `Debug` and +/// holds the private key, so it is consumed here and never formatted. +/// +/// `issuing_ca` rather than `ca_chain` because it is the one certificate a +/// cert-auth role pins: the authority that signed this leaf. +fn credential_from_issue(issued: GenerateCertificateResponse) -> Result { + for (field, value) in [ + ("certificate", &issued.certificate), + ("private_key", &issued.private_key), + ("issuing_ca", &issued.issuing_ca), + ] { + if value.trim().is_empty() { + return Err(Error::IncompleteIssue(field)); + } + } + Ok(mtls::Credential { + cert: issued.certificate, + key: issued.private_key, + ca: issued.issuing_ca, + }) +} + #[cfg(test)] mod tests { use super::*; @@ -408,4 +474,84 @@ mod tests { the store's copy of the document disagree with its own name" ); } + + /// The controller's grant names exactly this path with `update`, which is + /// what a POST needs; any other path or verb is a 403. + #[test] + fn an_issue_request_posts_to_the_roles_issue_path() { + use rustify::endpoint::Endpoint as _; + + let request = issue_request("pki-agents", "swarm-agent", "hive-agent-atlas"); + assert_eq!(request.path(), "pki-agents/issue/swarm-agent"); + assert!( + matches!(request.method(), rustify::enums::RequestMethod::POST), + "{:?}", + request.method() + ); + } + + #[test] + fn an_issue_request_asks_for_the_name_and_nothing_the_role_decides() { + use rustify::endpoint::Endpoint as _; + + let body = issue_request("pki-agents", "swarm-agent", "hive-agent-atlas") + .body() + .expect("the body serialises") + .expect("an issue request sends one"); + let sent: serde_json::Value = serde_json::from_slice(&body).expect("JSON"); + assert_eq!(sent["common_name"], "hive-agent-atlas"); + assert_eq!(sent["exclude_cn_from_sans"], true); + assert_eq!(sent["private_key_format"], "pkcs8"); + for decided_by_the_role in ["ttl", "alt_names", "ip_sans", "uri_sans", "other_sans"] { + assert!( + sent.get(decided_by_the_role) + .is_none_or(serde_json::Value::is_null), + "{decided_by_the_role} is the role's to set: {sent}" + ); + } + } + + fn issued() -> GenerateCertificateResponse { + GenerateCertificateResponse { + ca_chain: None, + certificate: "LEAF".to_owned(), + expiration: None, + issuing_ca: "AGENT-CA".to_owned(), + private_key: "SECRET-KEY-BYTES".to_owned(), + private_key_type: "ec".to_owned(), + serial_number: "01".to_owned(), + } + } + + #[test] + fn a_complete_answer_becomes_the_published_credential() { + let credential = credential_from_issue(issued()).expect("every field is present"); + assert_eq!(credential.cert, "LEAF"); + assert_eq!(credential.key, "SECRET-KEY-BYTES"); + assert_eq!(credential.ca, "AGENT-CA", "the role pins the issuing CA"); + assert!( + !format!("{credential:?}").contains("SECRET-KEY-BYTES"), + "the key must not reach a formatted value" + ); + } + + #[test] + fn an_answer_missing_any_part_of_the_identity_is_refused_by_name() { + type Blank = fn(&mut GenerateCertificateResponse); + let cases: [(&str, Blank); 3] = [ + ("certificate", |r| r.certificate.clear()), + ("private_key", |r| r.private_key = " \n".to_owned()), + ("issuing_ca", |r| r.issuing_ca.clear()), + ]; + for (field, blank) in cases { + let mut answer = issued(); + blank(&mut answer); + let e = credential_from_issue(answer).expect_err("an incomplete identity"); + assert!( + matches!(e, Error::IncompleteIssue(f) if f == field), + "blanking {field} gave {e:?}" + ); + assert!(!e.to_string().contains("SECRET-KEY-BYTES"), "{e}"); + } + } } diff --git a/swarm-secret-client/src/forge.rs b/swarm-secret-client/src/forge.rs index 8a0f68ce..a3319113 100644 --- a/swarm-secret-client/src/forge.rs +++ b/swarm-secret-client/src/forge.rs @@ -17,7 +17,7 @@ use crate::{ /// /// A flat leaf under the agent's prefix, like its controller-minted siblings /// [`crate::queue::agent_queue_path`] and [`crate::mtls::identity_path`], so -/// the agent's own read stanza ([`crate::policy::render_agent_with_queue`]) +/// the agent's own read stanza ([`crate::policy::render_agent`]) /// already covers it. /// /// # Errors @@ -89,7 +89,7 @@ mod tests { // policy is rendered elsewhere. If the path ever moved out from under // it, the agent's fetch would 403 at boot, naming neither. let path = agent_token_path("atlas").expect("legal"); - let policy = crate::policy::render_agent_with_queue("atlas", "pr1ma").expect("legal"); + let policy = crate::policy::render_agent("atlas").expect("legal"); let covered = policy.lines().any(|line| { line.strip_prefix("path \"") .and_then(|rest| rest.split_once("\" {")) @@ -104,7 +104,7 @@ mod tests { // Control for the test above: the prefix match must actually be // discriminating, or it proves nothing. let path = agent_token_path("atlas").expect("legal"); - let policy = crate::policy::render_agent_with_queue("argus", "pr1ma").expect("legal"); + let policy = crate::policy::render_agent("argus").expect("legal"); let covered = policy.lines().any(|line| { line.strip_prefix("path \"") .and_then(|rest| rest.split_once("\" {")) diff --git a/swarm-secret-client/src/lib.rs b/swarm-secret-client/src/lib.rs index cb93e056..6d7bccb2 100644 --- a/swarm-secret-client/src/lib.rs +++ b/swarm-secret-client/src/lib.rs @@ -80,6 +80,11 @@ pub enum Error { /// CA bundle did not parse. #[error("building the TLS identity: {0}")] Tls(#[source] reqwest::Error), + + /// The store answered an issue request with a field empty that a usable + /// identity needs. Names the field, never its value. + #[error("the store issued a certificate whose {0} is empty")] + IncompleteIssue(&'static str), } impl From for Error { diff --git a/swarm-secret-client/src/policy.rs b/swarm-secret-client/src/policy.rs index 72cdfa0c..f8b4460b 100644 --- a/swarm-secret-client/src/policy.rs +++ b/swarm-secret-client/src/policy.rs @@ -70,13 +70,12 @@ pub fn hive_object_name(hive: &str) -> Result { /// client ids; this is a second identifier family leaning on it, which is why /// the fragment list is what to read before renaming either. /// -/// ⚠️ The controller's and publisher's subjects are *not* covered by that: their -/// policy names are literals outside `hive-`, but their **common names** are -/// operator-set options (`deploy.bao.controllerCommonName`, -/// `secretPublisherCommonName`) that nothing here can see, and an operator may -/// spell one `hive-agent-atlas`. Whichever change first mints an agent leaf -/// owes the assertion that neither starts with this prefix — `swarm.nix`'s -/// `certAuthCns` is where the mirror-image check for hive names lives. +/// Host principals' **common names** are operator-set options +/// (`deploy.bao.controllerCommonName`, `secretPublisherCommonName`, …) that +/// may spell this prefix, and that is harmless: an agent's cert-auth role pins +/// the store's agent CA (`deploy.bao.agentPkiMountPath`), which signs no host +/// leaf, and every host role pins `deploy.bao.clientCaFile`, which signs no +/// agent leaf. A CN match alone logs nobody in. pub const AGENT_PREFIX: &str = "hive-agent-"; /// The policy and cert-auth role name for `agent`, and the common name of the @@ -203,43 +202,6 @@ pub fn render_agent(agent: &str) -> Result { ))) } -/// Render `agent`'s policy document: read on that one agent's credentials and -/// on the hive's shared queue credential. -/// -/// Extends [`render_agent`] with a second stanza granting read on -/// `swarm/hives//queue/agent`. The queue credential is **hive-shared, -/// not per-agent** — every agent in a hive authenticates to the queue with the -/// same client secret (`queue.rs:1-8`), so a policy scoped strictly to -/// `agents//*` cannot read it and an in-container pull would fail. That -/// hive-shared credential is already handed to every agent container on that -/// hive by the host today, so this grant adds no new authority — it merely -/// makes the existing capability reachable through the agent's own token -/// instead of requiring the credential to be delivered out of band. -/// -/// ⚠️ **Every agent in a hive can read that hive's queue credential.** This is -/// not new authority (the host already provides this exact value to all agents -/// on the hive), but it is a documented property: an agent policy grants read -/// on a path shared across every agent on its hive, not on a path unique to -/// that agent alone. -/// -/// Read-only, for the same reason [`render_agent`]'s is: an agent that could -/// write credentials could hand itself an identity it was never issued. -/// -/// # Errors -/// [`Error::PathSegment`] when `agent` or `hive` holds anything but -/// `[A-Za-z0-9_-]` — both are interpolated into policy paths, so a name that -/// could close a stanza could grant itself anything. -pub fn render_agent_with_queue(agent: &str, hive: &str) -> Result { - checked_segment("agent", agent)?; - let agent_stanza = read_stanza(&format!( - "{MOUNT}/data/{ROOT}/{}/{agent}/*", - <&str>::from(Kind::Agent) - )); - let queue_path = crate::queue::agent_client_path(hive)?; - let queue_stanza = read_stanza(&format!("{MOUNT}/data/{queue_path}")); - Ok(format!("{agent_stanza}{queue_stanza}")) -} - #[cfg(test)] mod tests { use super::*; @@ -430,6 +392,22 @@ mod tests { ); } + #[test] + fn an_agents_document_is_exactly_its_own_read_stanza() { + // Pinned byte for byte: an added stanza (a hive's queue credential, a + // second agent) is exactly what a presence check misses. + assert_eq!( + render_agent("atlas").expect("legal"), + "path \"secret/data/swarm/agents/atlas/*\" {\n capabilities = [\"read\"]\n}\n" + ); + // The control: the pin discriminates between agents. + assert!( + !render_agent("other") + .expect("legal") + .contains("agents/atlas/") + ); + } + #[test] fn an_agents_grant_is_read_only() { // An agent that could write its own credentials could hand itself an @@ -517,102 +495,6 @@ mod tests { assert!(hive.contains("path \"secret/data/swarm/agents/*\"")); } - #[test] - fn an_agents_document_with_queue_grants_both_paths() { - // The happy path: the document grants read on the agent's own namespace - // and on the hive's queue credential. - let p = render_agent_with_queue("atlas", "pr1ma").expect("legal"); - assert!( - p.contains("path \"secret/data/swarm/agents/atlas/*\""), - "must grant the agent's own path: {p}" - ); - let expected_queue_path = format!( - "path \"{MOUNT}/data/{}\"", - crate::queue::agent_client_path("pr1ma").expect("legal") - ); - assert!( - p.contains(&expected_queue_path), - "must grant the hive's queue credential: {p}" - ); - assert_eq!( - p.matches("path \"").count(), - 2, - "two stanzas, one for the agent and one for the queue: {p}" - ); - } - - #[test] - fn an_agents_document_with_queue_is_read_only() { - // An agent that could write the queue credential could hand every agent - // on its hive an identity they were never issued. - let p = render_agent_with_queue("atlas", "pr1ma").expect("legal"); - for capability in ["create", "update", "delete", "list", "sudo", "patch"] { - assert!(!p.contains(capability), "must not grant {capability}: {p}"); - } - assert!(p.contains("capabilities = [\"read\"]")); - } - - #[test] - fn an_agent_name_with_traversal_in_the_queue_variant_is_refused() { - // The agent parameter is an injection surface in both renderers, so - // refusing a traversal here proves the new one validates it. - assert!( - render_agent_with_queue("atlas/*\" { capabilities = [\"root\"] }", "pr1ma").is_err() - ); - assert!(render_agent_with_queue("", "pr1ma").is_err()); - // The control: legal names still work. - assert!(render_agent_with_queue("a-b_C9", "pr1ma").is_ok()); - } - - #[test] - fn a_hive_name_with_traversal_in_the_queue_variant_is_refused() { - // The hive parameter is a second injection surface that only the queue - // variant introduces, so this test proves that new parameter is - // validated. A name that could close the stanza could grant the agent - // anything. - assert!( - render_agent_with_queue("atlas", "pr1ma/*\" { capabilities = [\"root\"] }").is_err() - ); - assert!(render_agent_with_queue("atlas", "").is_err()); - assert!( - render_agent_with_queue("atlas", "../services/swarm-grafana").is_err(), - "a path traversal that could reach a different kind" - ); - // The control: legal names still work. - assert!(render_agent_with_queue("atlas", "a-b_C9").is_ok()); - } - - #[test] - fn the_queue_variant_does_not_widen_the_agent_stanza() { - // The queue grant must not cause the agent stanza to widen from - // `agents//*` to `agents/*` — that would give every agent every - // other agent's credentials. - let p = render_agent_with_queue("atlas", "pr1ma").expect("legal"); - assert!( - !p.contains("swarm/agents/*"), - "must not grant the whole agent prefix: {p}" - ); - assert!(p.contains("swarm/agents/atlas/*")); - } - - #[test] - fn the_queue_variant_does_not_grant_the_whole_hive_prefix() { - // The queue stanza must grant only the queue credential path, not - // `hives//*` — the latter would give the agent read on every - // secret of the hive that hosts it. - let p = render_agent_with_queue("atlas", "pr1ma").expect("legal"); - assert!( - !p.contains("swarm/hives/*"), - "must not grant the whole hive prefix: {p}" - ); - assert!( - !p.contains("swarm/hives/pr1ma/*"), - "must not grant the hive's whole path: {p}" - ); - let expected_queue_path = crate::queue::agent_client_path("pr1ma").expect("legal"); - assert!(p.contains(&expected_queue_path)); - } - #[test] fn only_agent_roles_come_back_and_without_their_prefix() { let roles: Vec = [