swarm: mint, publish and login-verify an agent's store identity at create
`swarm/agents/<agent>/bao-mtls` did not exist, and neither did any per-agent identity at the secret store: `policy::agent_object_name`, `render_agent` and `render_agent_with_queue` had been written and never called outside their own tests. An agent's only "per-agent" secret today is read under the HIVE's certificate, through a wide grant on `swarm/agents/*` — so "per-agent" was presentational. The swarm now mints the certificate, so no hive ever needs the capability to mint one. `swarm-controller` is the service that does it: it already logs in to the store, and its existing grant already covers exactly the three objects written here (`create/update` on `secret/data/swarm/agents/*`, `sys/policies/acl/hive-*` and `auth/cert/certs/hive-*`). No new bao grant, and nothing co-located — a cert-auth role pins its authority by value, per role, so the controller issues from its own CA on its own host and pins that CA in the role it writes. No existing role changes. The mint node does not report success on a write. After publishing it connects again, with the leaf it just issued and under the role it just wrote, and reads the path back — so the policy, the role, the common name and the leaf are exercised in production on every agent creation. A certificate this code mints that the role this code writes will not accept turns the job node red at creation time instead of surfacing later as an agent container that cannot start. `TriggerDeploy` gains an `after_any` edge on the mint, not `after_ok`: a hive cannot pass down a certificate the swarm has not published, but a host with no authority configured must still create agents exactly as it does today. The private key is generated in memory and never written to disk on the controller — `SecretStore::connect_with_identity` takes the PEM the minter is already holding, so nothing is written out purely to be logged in with. Refs #4137
This commit is contained in:
atlas
parent
992468dccc
commit
676c45bc93
10 changed files with 1262 additions and 71 deletions
203
Cargo.lock
generated
203
Cargo.lock
generated
|
|
@ -162,6 +162,45 @@ version = "1.3.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9dbc3a507a82b17ba0d98f6ce8fd6954ea0c8152e98009d36a40d8dcc8ce078a"
|
||||
|
||||
[[package]]
|
||||
name = "asn1-rs"
|
||||
version = "0.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8"
|
||||
dependencies = [
|
||||
"asn1-rs-derive",
|
||||
"asn1-rs-impl",
|
||||
"displaydoc",
|
||||
"nom",
|
||||
"num-traits",
|
||||
"rusticata-macros",
|
||||
"thiserror 2.0.18",
|
||||
"time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "asn1-rs-derive"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
"synstructure 0.13.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "asn1-rs-impl"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "assign"
|
||||
version = "1.1.1"
|
||||
|
|
@ -198,7 +237,7 @@ version = "0.50.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d83a251fa1a4c9d0fe6e816b7acd60549e473e08d14f27a1d992c2675abff05f"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"memchr",
|
||||
|
|
@ -308,7 +347,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90"
|
||||
dependencies = [
|
||||
"axum-core",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"form_urlencoded",
|
||||
"futures-util",
|
||||
|
|
@ -373,6 +412,12 @@ version = "0.22.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.23.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
|
||||
|
||||
[[package]]
|
||||
name = "base64ct"
|
||||
version = "1.8.3"
|
||||
|
|
@ -385,13 +430,22 @@ version = "0.19.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f3c067aa24dd4ed5c79cf222a38f260c8f23d3b82a062fba3f28c6fe563b753"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"blowfish",
|
||||
"getrandom 0.4.3",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bit-vec"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.13.1"
|
||||
|
|
@ -1035,6 +1089,20 @@ dependencies = [
|
|||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "der-parser"
|
||||
version = "10.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6"
|
||||
dependencies = [
|
||||
"asn1-rs",
|
||||
"displaydoc",
|
||||
"nom",
|
||||
"num-bigint",
|
||||
"num-traits",
|
||||
"rusticata-macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "deranged"
|
||||
version = "0.5.8"
|
||||
|
|
@ -1740,7 +1808,7 @@ dependencies = [
|
|||
"anyhow",
|
||||
"async-nats",
|
||||
"axum",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bcrypt",
|
||||
"chrono",
|
||||
"clap",
|
||||
|
|
@ -2190,7 +2258,7 @@ version = "0.1.20"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
|
|
@ -2903,7 +2971,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "fef37395fffb7c916f7109ab0d16d8ca599403dd8164d08c0d966176b66ede47"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"futures-util",
|
||||
"getrandom 0.4.3",
|
||||
"gloo-utils",
|
||||
|
|
@ -2962,7 +3030,7 @@ version = "0.18.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2f48f304e553fb6200b1d7d1f77a88fd182076d4b25624e9dbfa42d6a37de35e"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"blake3",
|
||||
"chacha20poly1305",
|
||||
"getrandom 0.2.17",
|
||||
|
|
@ -3045,6 +3113,12 @@ dependencies = [
|
|||
"unicase",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "minimal-lexical"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.8.9"
|
||||
|
|
@ -3100,6 +3174,16 @@ dependencies = [
|
|||
"signatory",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nom"
|
||||
version = "7.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
"minimal-lexical",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nu-ansi-term"
|
||||
version = "0.50.3"
|
||||
|
|
@ -3109,12 +3193,31 @@ dependencies = [
|
|||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-bigint"
|
||||
version = "0.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367"
|
||||
dependencies = [
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-conv"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441"
|
||||
|
||||
[[package]]
|
||||
name = "num-integer"
|
||||
version = "0.1.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.19"
|
||||
|
|
@ -3140,7 +3243,7 @@ version = "5.0.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"chrono",
|
||||
"getrandom 0.2.17",
|
||||
"http",
|
||||
|
|
@ -3163,6 +3266,15 @@ dependencies = [
|
|||
"reqwest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "oid-registry"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7"
|
||||
dependencies = [
|
||||
"asn1-rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.4"
|
||||
|
|
@ -3237,7 +3349,7 @@ version = "0.32.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"const-hex",
|
||||
"opentelemetry",
|
||||
"opentelemetry_sdk",
|
||||
|
|
@ -3311,6 +3423,16 @@ dependencies = [
|
|||
"digest 0.10.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pem"
|
||||
version = "4.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120"
|
||||
dependencies = [
|
||||
"base64 0.23.1",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pem-rfc7468"
|
||||
version = "0.7.0"
|
||||
|
|
@ -3754,6 +3876,20 @@ dependencies = [
|
|||
"rand_core 0.9.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rcgen"
|
||||
version = "0.14.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8"
|
||||
dependencies = [
|
||||
"pem",
|
||||
"ring",
|
||||
"rustls-pki-types",
|
||||
"time",
|
||||
"x509-parser",
|
||||
"yasna",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "readlock"
|
||||
version = "0.1.11"
|
||||
|
|
@ -3833,7 +3969,7 @@ version = "0.13.4"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"encoding_rs",
|
||||
"futures-channel",
|
||||
|
|
@ -3995,7 +4131,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "2c3b4f00112791b490acce57df1ce3eb3f88899b045bebcff8a29f75369640cc"
|
||||
dependencies = [
|
||||
"as_variant",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"date_header",
|
||||
"form_urlencoded",
|
||||
|
|
@ -4112,6 +4248,15 @@ dependencies = [
|
|||
"semver",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rusticata-macros"
|
||||
version = "4.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632"
|
||||
dependencies = [
|
||||
"nom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustify"
|
||||
version = "0.7.0"
|
||||
|
|
@ -4734,7 +4879,7 @@ dependencies = [
|
|||
"async-nats",
|
||||
"async-trait",
|
||||
"axum",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"forgejo-api",
|
||||
"futures-util",
|
||||
|
|
@ -4749,6 +4894,7 @@ dependencies = [
|
|||
"opentelemetry-otlp",
|
||||
"opentelemetry_sdk",
|
||||
"problem_details",
|
||||
"rcgen",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
|
|
@ -4757,6 +4903,7 @@ dependencies = [
|
|||
"swarm-authelia-bridge-sock",
|
||||
"swarm-queue-client",
|
||||
"swarm-secret-client",
|
||||
"time",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
|
|
@ -5137,7 +5284,7 @@ version = "0.10.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f591660438b3038dd04d16c938271c79e7e06260ad2ea2885a4861bfb238605d"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
|
|
@ -5579,7 +5726,7 @@ checksum = "b98bf83c0992966775b8012f194b07b44928996163e5a05b741b43891571ae5b"
|
|||
dependencies = [
|
||||
"aes",
|
||||
"arrayvec",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"base64ct",
|
||||
"cbc",
|
||||
"chacha20poly1305",
|
||||
|
|
@ -6069,12 +6216,40 @@ dependencies = [
|
|||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "x509-parser"
|
||||
version = "0.18.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202"
|
||||
dependencies = [
|
||||
"asn1-rs",
|
||||
"data-encoding",
|
||||
"der-parser",
|
||||
"lazy_static",
|
||||
"nom",
|
||||
"oid-registry",
|
||||
"ring",
|
||||
"rusticata-macros",
|
||||
"thiserror 2.0.18",
|
||||
"time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "xxhash-rust"
|
||||
version = "0.8.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "985eec839aaf2a1270af8f4ebcf63cf9401cfd90f0902f97c28d9f104ffbde72"
|
||||
|
||||
[[package]]
|
||||
name = "yasna"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282"
|
||||
dependencies = [
|
||||
"bit-vec",
|
||||
"time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "yoke"
|
||||
version = "0.8.3"
|
||||
|
|
|
|||
14
Cargo.toml
14
Cargo.toml
|
|
@ -101,6 +101,20 @@ vaultrs = "0.8"
|
|||
# resolves, since its `exec_with_empty` takes *its* `Endpoint` trait.
|
||||
rustify = "0.7"
|
||||
rustify_derive = "0.5"
|
||||
# Signs the per-agent client leaf `swarm-controller::agent_identity` mints.
|
||||
# A library rather than an `openssl` shellout, which is what every other CA
|
||||
# in this tree is (`glue-bao-tls.nix`, `hive-tls.nix`, `swarm-ca.nix`): those
|
||||
# run once at deploy time and write to disk, this one runs per agent
|
||||
# creation and must keep the private key it generates in memory long enough
|
||||
# to log in with it and no longer. `ring` over `aws_lc_rs` because `ring` is
|
||||
# already in the lock; the crypto backend is not otherwise load-bearing.
|
||||
#
|
||||
# `x509-parser`: `Issuer::from_ca_cert_pem` is gated behind it, and reading
|
||||
# the authority back out of its own PEM is how a leaf inherits the subject and
|
||||
# key identifier that make it chain. The ungated constructors take a
|
||||
# `CertificateParams` the caller would have to restate by hand — two spellings
|
||||
# of one authority, agreeing until the day they don't.
|
||||
rcgen = { version = "0.14", features = ["x509-parser"] }
|
||||
tower-http = { version = "0.7", features = ["fs"] }
|
||||
uuid = { version = "1", features = ["v4"] }
|
||||
rmcp = { version = "2", default-features = false, features = [
|
||||
|
|
|
|||
|
|
@ -48,13 +48,24 @@ path at runtime — not a path on disk, and not a unit whose job is to turn a
|
|||
store value into a file. A renewal cell may never read `NONE`: state the
|
||||
strategy for every credential, including the mTLS leaf.
|
||||
|
||||
| store path | minter | reader — pulls at runtime, holds in memory | renewal |
|
||||
| -------------------------------------------- | ------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------ | -------------- |
|
||||
| `swarm/agents/<agent>/matrix/<account>` | `swarm-controller` | the agent container itself, under the certificate its hive passed in | must be stated |
|
||||
| `swarm/hives/<hive>/matrix/appservice-token` | one minter, on the authelia host | the hive process that presents the token to its homeserver, under the hive's own certificate | must be stated |
|
||||
| `swarm/hives/<hive>/queue/agent` | authelia | the agent container presenting the OIDC client to the swarm queue, under its own certificate | must be stated |
|
||||
| `swarm/services/<clientId>/oidc/client` | authelia | the service process that presents the client secret, under the certificate of the host it runs on | must be stated |
|
||||
| _(not in the store)_ the mTLS client leaf | the store's own PKI, or an operator placing it by hand | its own client, off disk — the sole exception above, because it's what makes every other row's pull possible | must be stated |
|
||||
| store path | minter | reader — pulls at runtime, holds in memory | renewal |
|
||||
| -------------------------------------------- | ------------------------------------------------------ | ------------------------------------------------------------------------------------------------------- | -------------- |
|
||||
| `swarm/agents/<agent>/matrix/<account>` | `swarm-controller` | the agent container itself, under the certificate its hive passed in | must be stated |
|
||||
| `swarm/agents/<agent>/bao-mtls` | `swarm-controller`, at agent creation | the agent's **hive**, under the hive's own certificate, which hands it into the container | must be stated |
|
||||
| `swarm/hives/<hive>/matrix/appservice-token` | one minter, on the authelia host | the hive process that presents the token to its homeserver, under the hive's own certificate | must be stated |
|
||||
| `swarm/hives/<hive>/queue/agent` | authelia | the agent container presenting the OIDC client to the swarm queue, under its own certificate | must be stated |
|
||||
| `swarm/services/<clientId>/oidc/client` | authelia | the service process that presents the client secret, under the certificate of the host it runs on | must be stated |
|
||||
| _(not in the store)_ a hive's mTLS leaf | the store's own PKI, or an operator placing it by hand | its own client, off disk — the exception above, because it's what makes every other row's pull possible | must be stated |
|
||||
|
||||
**An agent's mTLS leaf is in the store; a hive's is not, and the difference
|
||||
is not an inconsistency.** The rule the exception protects is that nothing
|
||||
can fetch from the store the credential it would need in order to fetch. A
|
||||
hive's leaf is that credential, so it can only come off disk. An agent's is
|
||||
read by the _hive_, which already holds one — so publishing it costs nothing
|
||||
and buys the property this page asks for: the swarm mints it, the hive only
|
||||
carries it, and no hive ever needs the capability to mint an identity.
|
||||
`swarm-controller` proves the leaf it publishes before the creation job
|
||||
reports success, by logging in with it and reading the row back.
|
||||
|
||||
## Progressive enhancement
|
||||
|
||||
|
|
|
|||
|
|
@ -37,6 +37,33 @@ let
|
|||
# file would be handed to a daemon that cannot use it.
|
||||
haveHiveClientCa = haveBaoIdentity && deployCfg.swarm-controller.hiveClientCaFile != null;
|
||||
|
||||
# The authority this daemon issues AGENT client leaves from — a different
|
||||
# question from `hiveClientCaFile` above, which is the authority it *trusts*
|
||||
# hives by. This one it signs with, so it needs the private key too.
|
||||
#
|
||||
# ⚠️ Deliberately NOT the store's own PKI (`glue-bao-tls.nix`'s
|
||||
# `/var/lib/swarm-bao-pki`). A cert-auth role pins its authority by value,
|
||||
# per role, so a role this daemon writes carries whatever authority this
|
||||
# daemon hands it — which is what lets the controller mint from its own CA
|
||||
# on its own host without anything being co-located and without any
|
||||
# existing role changing. `swarm-controller/src/agent_identity.rs`'s module
|
||||
# doc is the long form.
|
||||
agentCaDir = "/var/lib/swarm-controller-agent-ca";
|
||||
|
||||
# No authority named means mint one here. The alternative — leaving agent
|
||||
# identities off until an operator places a CA by hand — is the state where
|
||||
# the whole path is configured and silently does nothing, which is the
|
||||
# failure mode `glue-bao-tls.nix` avoids the same way.
|
||||
selfSignAgentCa = deployCfg.swarm-controller.agentCaFile == null;
|
||||
agentCaCert =
|
||||
if selfSignAgentCa then "${agentCaDir}/ca.pem" else deployCfg.swarm-controller.agentCaFile;
|
||||
agentCaKey =
|
||||
if selfSignAgentCa then "${agentCaDir}/ca-key.pem" else deployCfg.swarm-controller.agentCaKeyFile;
|
||||
|
||||
# Minting an agent's identity means publishing it to the store, so the
|
||||
# authority alone is not enough — same rule `haveHiveClientCa` states.
|
||||
haveAgentCa = haveBaoIdentity && agentCaKey != null;
|
||||
|
||||
# `swarm_secret_client` reads these spellings explicitly rather than
|
||||
# vaultrs's `VAULT_*` defaults — falling through to those builds a client
|
||||
# with no identity and fails at the TLS handshake, naming neither. `%d` and
|
||||
|
|
@ -59,6 +86,13 @@ let
|
|||
# store should trust hives by, which the controller reads as a value
|
||||
# to put in each hive's cert-auth role.
|
||||
SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE = "%d/hive-client-ca.pem";
|
||||
}
|
||||
// lib.optionalAttrs haveAgentCa {
|
||||
# The authority agent leaves are ISSUED FROM, so unlike every other
|
||||
# `*_CA_FILE` here it comes with a key. `%d` for both: the key is
|
||||
# `0600` and root-owned, and this daemon runs unprivileged.
|
||||
SWARM_CONTROLLER_AGENT_CA_FILE = "%d/agent-ca.pem";
|
||||
SWARM_CONTROLLER_AGENT_CA_KEY_FILE = "%d/agent-ca-key.pem";
|
||||
};
|
||||
|
||||
# What `swarmctl` needs in order to act on authelia from the host.
|
||||
|
|
@ -604,6 +638,47 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
agentCaFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "/var/lib/swarm-agent-ca/ca.pem";
|
||||
description = ''
|
||||
Authority this daemon **issues** agent client certificates from, so
|
||||
that an agent container can authenticate to the swarm secret store
|
||||
under its own name. Read together with
|
||||
{option}`services.hyperhive.deploy.swarm-controller.agentCaKeyFile`,
|
||||
which is the private key it signs with.
|
||||
|
||||
The mirror image of
|
||||
{option}`services.hyperhive.deploy.swarm-controller.hiveClientCaFile`:
|
||||
that one is an authority this daemon only *trusts by value*, so it is
|
||||
public material and needs no key. This one signs, so it does.
|
||||
|
||||
Leaving this `null` — the default — makes the module mint a
|
||||
self-signed authority in `${agentCaDir}` on first boot and use that.
|
||||
That is the ordinary shape: the store pins an authority per cert-auth
|
||||
role, by value, so the authority agents are issued from does not have
|
||||
to be the store's own PKI and does not have to live on the store's
|
||||
host. Name a file here only when an operator issues agent leaves from
|
||||
somewhere else; doing so turns the self-signing unit off.
|
||||
'';
|
||||
};
|
||||
|
||||
agentCaKeyFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "/var/lib/swarm-agent-ca/ca-key.pem";
|
||||
description = ''
|
||||
Private key for
|
||||
{option}`services.hyperhive.deploy.swarm-controller.agentCaFile`.
|
||||
Both or neither — an authority with no key signs nothing, and the
|
||||
daemon refuses to start half-configured rather than looking ready.
|
||||
|
||||
A path, never a value: the key's bytes in a nix expression land in
|
||||
the world-readable nix store, permanently.
|
||||
'';
|
||||
};
|
||||
|
||||
queue = {
|
||||
clientSecretFile = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
|
|
@ -634,7 +709,10 @@ in
|
|||
services.hyperhive.swarm.otel.journaldUnits = [
|
||||
"swarm-controller"
|
||||
"swarm-controller-credential"
|
||||
];
|
||||
]
|
||||
# Declared only where the unit exists — an entry for a unit that was
|
||||
# never defined is a collector waiting on a journal that never speaks.
|
||||
++ lib.optional (haveAgentCa && selfSignAgentCa) "swarm-controller-agent-ca";
|
||||
|
||||
users.users.swarm-controller = {
|
||||
isSystemUser = true;
|
||||
|
|
@ -721,6 +799,18 @@ in
|
|||
state directory.
|
||||
'';
|
||||
}
|
||||
{
|
||||
assertion =
|
||||
deployCfg.swarm-controller.agentCaFile == null || deployCfg.swarm-controller.agentCaKeyFile != null;
|
||||
message = ''
|
||||
services.hyperhive.deploy.swarm-controller.agentCaFile names an
|
||||
authority but agentCaKeyFile is unset.
|
||||
|
||||
The controller does not merely trust this authority, it issues
|
||||
agent client certificates from it, so it needs the private key.
|
||||
Set both, or set neither and let the module mint its own.
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
systemd.services.swarm-controller = {
|
||||
|
|
@ -762,7 +852,16 @@ in
|
|||
++ lib.optional (
|
||||
haveBaoIdentity && deployCfg.bao.serverCaFile != null
|
||||
) "bao-ca.pem:${deployCfg.bao.serverCaFile}"
|
||||
++ lib.optional haveHiveClientCa "hive-client-ca.pem:${deployCfg.swarm-controller.hiveClientCaFile}";
|
||||
++ lib.optional haveHiveClientCa "hive-client-ca.pem:${deployCfg.swarm-controller.hiveClientCaFile}"
|
||||
# The agent authority, key included — same shape and same reason as
|
||||
# the store identity above: the key is root-owned `0600` and this
|
||||
# daemon runs as `swarm-controller`. `swarm-controller-agent-ca`
|
||||
# below is `requiredBy` this unit, so the files exist by the time
|
||||
# systemd resolves these.
|
||||
++ lib.optionals haveAgentCa [
|
||||
"agent-ca.pem:${agentCaCert}"
|
||||
"agent-ca-key.pem:${agentCaKey}"
|
||||
];
|
||||
|
||||
# The placeholder default that makes the above non-fatal.
|
||||
# `LoadCredential=` takes priority over `SetCredential=`, so this is
|
||||
|
|
@ -935,5 +1034,50 @@ in
|
|||
ExecStart = "${pkgs.systemd}/bin/systemctl try-restart swarm-controller.service";
|
||||
};
|
||||
};
|
||||
|
||||
# The authority agent client leaves are issued from, minted here when the
|
||||
# operator named none. Shape copied from ./glue-bao-tls.nix's
|
||||
# `swarm-bao-pki`, including the rule that matters most:
|
||||
#
|
||||
# 🩸 Idempotent on ABSENCE, never on content. Re-issuing this CA would
|
||||
# invalidate every agent leaf already published to the store AND every
|
||||
# cert-auth role that pinned it by value, locking every agent container
|
||||
# in the swarm out at once — on a rebuild that changed nothing an
|
||||
# operator asked for.
|
||||
#
|
||||
# `before` + `requiredBy` rather than `after`: the daemon's
|
||||
# `LoadCredential=` names these files by absolute path, and a
|
||||
# `LoadCredential=` pointing at a file that is not there yet is fatal
|
||||
# (`243/CREDENTIALS`), not a slow start.
|
||||
systemd.services.swarm-controller-agent-ca = lib.mkIf (haveAgentCa && selfSignAgentCa) {
|
||||
description = "mint the authority swarm agents' store certificates are issued from";
|
||||
before = [ "swarm-controller.service" ];
|
||||
requiredBy = [ "swarm-controller.service" ];
|
||||
path = [
|
||||
pkgs.openssl
|
||||
pkgs.coreutils
|
||||
];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
};
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
install -d -m 0700 ${agentCaDir}
|
||||
|
||||
if [ ! -s ${agentCaCert} ]; then
|
||||
# `pathlen:0` — this authority signs leaves and nothing else. An
|
||||
# intermediate under it would be a second issuer for the one name
|
||||
# space the store matches agents by.
|
||||
openssl req -x509 -newkey rsa:4096 -nodes -sha256 -days 3650 \
|
||||
-keyout ${agentCaKey} -out ${agentCaCert} \
|
||||
-subj "/CN=swarm-agent-ca ${swarmDomain}" \
|
||||
-addext "basicConstraints=critical,CA:TRUE,pathlen:0" \
|
||||
-addext "keyUsage=critical,keyCertSign,cRLSign"
|
||||
chmod 0600 ${agentCaKey}
|
||||
chmod 0644 ${agentCaCert}
|
||||
fi
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -91,6 +91,15 @@ swarm-queue-client = { workspace = true, features = ["kv"] }
|
|||
# name and the object's shape are agreements between the two ends, and a
|
||||
# second spelling here would be a store this hive could not read.
|
||||
swarm-secret-client.workspace = true
|
||||
# `agent_identity.rs` signs the per-agent client leaf the store authenticates
|
||||
# an agent container by. The one runtime signer in this tree — every other CA
|
||||
# here is a deploy-time `openssl` oneshot — because this one issues per agent
|
||||
# creation and must keep the key it generates in memory, never on disk.
|
||||
rcgen.workspace = true
|
||||
# `rcgen::CertificateParams`'s validity window is a `time::OffsetDateTime`,
|
||||
# which `agent_identity::validity` builds. Same workspace pin every other
|
||||
# holder of a timestamp here uses.
|
||||
time.workspace = true
|
||||
# `otel_http_client.rs`'s `AuthenticatedHttpClient` — an
|
||||
# `opentelemetry_http::HttpClient` impl authenticated with this crate's own
|
||||
# `swarm-queue-client` identity. Lives in this crate rather than
|
||||
|
|
|
|||
420
swarm-controller/src/agent_identity.rs
Normal file
420
swarm-controller/src/agent_identity.rs
Normal file
|
|
@ -0,0 +1,420 @@
|
|||
//! One agent's own identity at the swarm's secret store: minted here,
|
||||
//! published here, granted here — and, before the job node reports success,
|
||||
//! **used** here.
|
||||
//!
|
||||
//! The swarm mints the agent's certificate so that no hive ever needs the
|
||||
//! capability to mint one; the hive only carries it down. The controller is
|
||||
//! the swarm-level service that does it because it already logs in to the
|
||||
//! store, and its grant already covers exactly the three objects written here
|
||||
//! (`swarm-bao.nix`'s `controllerPolicyText`: `create/update` on
|
||||
//! `secret/data/swarm/agents/*`, on `sys/policies/acl/hive-*`, and on
|
||||
//! `auth/cert/certs/hive-*`). No new authority is asked for anywhere.
|
||||
//!
|
||||
//! Four separate strings have to agree before an agent can authenticate: the
|
||||
//! policy's name, the cert-auth role's name, the certificate's common name,
|
||||
//! and the authority the role pins. That is the kind of agreement that holds
|
||||
//! in review and fails in production — a mismatch is a 403 naming none of the
|
||||
//! four — so [`mint_and_verify`] does not finish on a write. See
|
||||
//! [`read_back_as_agent`].
|
||||
//!
|
||||
//! ⚠️ The authority is **not** `/var/lib/swarm-bao-pki/ca-key.pem`. A
|
||||
//! cert-auth role pins its authority by value, per role (see
|
||||
//! [`SecretStore::write_cert_role`][swarm_secret_client::SecretStore::write_cert_role]),
|
||||
//! so a role this daemon writes carries whatever authority this daemon hands
|
||||
//! it — which is what lets the controller mint from its own CA on its own
|
||||
//! host, with nothing co-located and no existing role changed.
|
||||
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use swarm_secret_client::{
|
||||
SecretStore,
|
||||
client::{DEFAULT_CERT_MOUNT, Settings},
|
||||
mtls, policy,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
/// File holding the authority agent leaves are issued from, as
|
||||
/// `swarm-controller.nix` names it. Public material.
|
||||
pub const ENV_AGENT_CA: &str = "SWARM_CONTROLLER_AGENT_CA_FILE";
|
||||
|
||||
/// File holding the private key for [`ENV_AGENT_CA`]. 🩸 A path, never a
|
||||
/// value — the key's bytes must not reach a unit file or the nix store.
|
||||
pub const ENV_AGENT_CA_KEY: &str = "SWARM_CONTROLLER_AGENT_CA_KEY_FILE";
|
||||
|
||||
/// How long a minted leaf is good for.
|
||||
///
|
||||
/// Short enough that a leaked key is not permanent, long enough that the
|
||||
/// absence of a renewal path is not immediately fatal. Nothing re-mints a leaf
|
||||
/// today, so until a renewal path lands this is the interval after which an
|
||||
/// operator re-runs agent creation. It is deliberately far shorter than the ten
|
||||
/// years
|
||||
/// `glue-bao-tls.nix` gives the store's own CA: that one is an authority
|
||||
/// whose reissue invalidates every leaf under it, this one is a leaf.
|
||||
/// Spelled in hours because `Duration::from_days` is not yet a stable `const
|
||||
/// fn`; `read_policy`'s `RETRY_WINDOW` is the same workaround.
|
||||
const LEAF_LIFETIME: Duration = Duration::from_hours(90 * 24);
|
||||
|
||||
/// How far a leaf is backdated.
|
||||
///
|
||||
/// The verifier is the store, on another machine: a certificate whose
|
||||
/// `notBefore` is this exact instant is refused outright by a clock a second
|
||||
/// behind ours, and the resulting error names a validity window rather than a
|
||||
/// clock.
|
||||
const CLOCK_SKEW: Duration = Duration::from_mins(5);
|
||||
|
||||
/// The authority this daemon issues agent leaves from, loaded once at startup.
|
||||
///
|
||||
/// ⚠️ **No `Debug` derive**, and the key field is private: this struct is
|
||||
/// reachable from `WorkerDeps`, which is formatted nowhere today and is one
|
||||
/// `#[derive(Debug)]` away from being formatted everywhere.
|
||||
pub struct Authority {
|
||||
/// The authority's certificate, PEM. Public material — it is also what
|
||||
/// goes into each agent's cert-auth role and into each agent's published
|
||||
/// credential.
|
||||
ca_pem: String,
|
||||
/// The authority's private key, PEM. Never logged, never published,
|
||||
/// never leaves this struct.
|
||||
key_pem: String,
|
||||
}
|
||||
|
||||
impl Authority {
|
||||
/// Load the authority from the files [`ENV_AGENT_CA`] and
|
||||
/// [`ENV_AGENT_CA_KEY`] name, or `None` when this host was given neither.
|
||||
///
|
||||
/// `None` is a supported deployment, not a failure: it is the state every
|
||||
/// controller is in before an operator has turned agent identities on, and
|
||||
/// `run_swarm_node` reports it as that one node's named failure rather
|
||||
/// than refusing to start the daemon.
|
||||
///
|
||||
/// # Errors
|
||||
/// When exactly one of the two variables is set — half an authority signs
|
||||
/// nothing, and silently doing nothing about it is how a host ends up
|
||||
/// looking configured — or when a named file cannot be read.
|
||||
pub fn from_env() -> Result<Option<Self>> {
|
||||
match (
|
||||
std::env::var_os(ENV_AGENT_CA),
|
||||
std::env::var_os(ENV_AGENT_CA_KEY),
|
||||
) {
|
||||
(None, None) => Ok(None),
|
||||
(Some(_), None) => bail!(
|
||||
"{ENV_AGENT_CA} is set but {ENV_AGENT_CA_KEY} is not — an authority with no key signs nothing"
|
||||
),
|
||||
(None, Some(_)) => bail!(
|
||||
"{ENV_AGENT_CA_KEY} is set but {ENV_AGENT_CA} is not — a key with no certificate is not an authority"
|
||||
),
|
||||
(Some(ca), Some(key)) => {
|
||||
let ca_path = ca.to_string_lossy().into_owned();
|
||||
let key_path = key.to_string_lossy().into_owned();
|
||||
Ok(Some(Self {
|
||||
ca_pem: std::fs::read_to_string(&ca_path).with_context(|| {
|
||||
format!("reading the agent authority {ca_path} (from {ENV_AGENT_CA})")
|
||||
})?,
|
||||
key_pem: std::fs::read_to_string(&key_path).with_context(|| {
|
||||
format!(
|
||||
"reading the agent authority's key {key_path} (from {ENV_AGENT_CA_KEY})"
|
||||
)
|
||||
})?,
|
||||
}))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Build one from PEM already in hand — the constructor a test uses, and
|
||||
/// the one that keeps [`Authority::from_env`] the only place this process
|
||||
/// reads a key off disk.
|
||||
#[cfg(test)]
|
||||
fn from_pem(ca_pem: String, key_pem: String) -> Self {
|
||||
Self { ca_pem, key_pem }
|
||||
}
|
||||
|
||||
/// Issue a client leaf carrying `common_name`, returning `(certificate,
|
||||
/// private key)` as PEM.
|
||||
///
|
||||
/// `clientAuth` and nothing else: this certificate authenticates a
|
||||
/// principal to the store and must not be usable to *serve* anything.
|
||||
///
|
||||
/// # Errors
|
||||
/// When the authority's own PEM will not parse, or the leaf will not sign.
|
||||
fn mint_leaf(&self, common_name: &str) -> Result<(String, String)> {
|
||||
let issuer_key = rcgen::KeyPair::from_pem(&self.key_pem)
|
||||
.context("the agent authority's key is not a PEM key that can sign")?;
|
||||
let issuer = rcgen::Issuer::from_ca_cert_pem(&self.ca_pem, issuer_key)
|
||||
.context("the agent authority is not a PEM certificate that can issue")?;
|
||||
|
||||
let (not_before, not_after) = validity(SystemTime::now(), LEAF_LIFETIME)?;
|
||||
let mut params = rcgen::CertificateParams::default();
|
||||
params.distinguished_name = rcgen::DistinguishedName::new();
|
||||
params
|
||||
.distinguished_name
|
||||
.push(rcgen::DnType::CommonName, common_name);
|
||||
params.is_ca = rcgen::IsCa::NoCa;
|
||||
params.use_authority_key_identifier_extension = true;
|
||||
params.key_usages = vec![
|
||||
rcgen::KeyUsagePurpose::DigitalSignature,
|
||||
rcgen::KeyUsagePurpose::KeyEncipherment,
|
||||
];
|
||||
params.extended_key_usages = vec![rcgen::ExtendedKeyUsagePurpose::ClientAuth];
|
||||
params.not_before = not_before;
|
||||
params.not_after = not_after;
|
||||
|
||||
let leaf_key = rcgen::KeyPair::generate().context("generating the leaf's key")?;
|
||||
let cert = params
|
||||
.signed_by(&leaf_key, &issuer)
|
||||
.with_context(|| format!("signing a leaf for {common_name}"))?;
|
||||
Ok((cert.pem(), leaf_key.serialize_pem()))
|
||||
}
|
||||
}
|
||||
|
||||
/// The validity window of a leaf minted at `now`, backdated by [`CLOCK_SKEW`].
|
||||
///
|
||||
/// A free function taking `now` rather than reading the clock itself, so the
|
||||
/// arithmetic — the part that can be wrong by a factor of sixty — is testable
|
||||
/// without waiting ninety days.
|
||||
///
|
||||
/// # Errors
|
||||
/// When the system clock is before the unix epoch, or so far past it that the
|
||||
/// window will not fit a timestamp.
|
||||
fn validity(now: SystemTime, lifetime: Duration) -> Result<(OffsetDateTime, OffsetDateTime)> {
|
||||
let secs = now
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.context("the system clock is before the unix epoch")?
|
||||
.as_secs();
|
||||
let secs = i64::try_from(secs).context("the system clock is past what a timestamp holds")?;
|
||||
let skew = i64::try_from(CLOCK_SKEW.as_secs()).expect("a five-minute constant fits an i64");
|
||||
let life = i64::try_from(lifetime.as_secs()).context("the leaf lifetime does not fit")?;
|
||||
|
||||
let not_before = OffsetDateTime::from_unix_timestamp(secs - skew)
|
||||
.context("the backdated start is not a representable time")?;
|
||||
let not_after = OffsetDateTime::from_unix_timestamp(secs + life)
|
||||
.context("the expiry is not a representable time")?;
|
||||
Ok((not_before, not_after))
|
||||
}
|
||||
|
||||
/// Give `agent` an identity at the store, and prove it works.
|
||||
///
|
||||
/// Four store writes' worth of agreement, then the login that checks it:
|
||||
///
|
||||
/// 1. mint a leaf whose common name is [`policy::agent_object_name`];
|
||||
/// 2. publish it at [`mtls::identity_path`], where the agent's hive collects
|
||||
/// it under the hive's own certificate;
|
||||
/// 3. write the ACL document [`policy::render_agent_with_queue`] renders —
|
||||
/// read on this one agent's paths, plus the hive-shared queue credential
|
||||
/// every agent container on `hive` already receives out of band;
|
||||
/// 4. write the cert-auth role that ties the three together.
|
||||
///
|
||||
/// Policy before role, for the reason `read_policy::provision` gives: the role
|
||||
/// names the policy, so the other order leaves a window in which it points at
|
||||
/// nothing.
|
||||
///
|
||||
/// # Errors
|
||||
/// Anything that stops one of those five steps, with the step named. A
|
||||
/// failure here fails the job node and nothing else — the agent is still
|
||||
/// created, exactly as capable as every agent is today.
|
||||
pub async fn mint_and_verify(authority: &Authority, agent: &str, hive: &str) -> Result<()> {
|
||||
let name = policy::agent_object_name(agent)?;
|
||||
let path = mtls::identity_path(agent)?;
|
||||
|
||||
let (cert, key) = authority.mint_leaf(&name)?;
|
||||
let credential = mtls::Credential {
|
||||
cert,
|
||||
key,
|
||||
ca: authority.ca_pem.clone(),
|
||||
};
|
||||
|
||||
let store = crate::store::connect()
|
||||
.await
|
||||
.context("logging in to the swarm secret store")?;
|
||||
store
|
||||
.write(&path, &credential)
|
||||
.await
|
||||
.with_context(|| format!("publishing the agent identity at {path}"))?;
|
||||
store
|
||||
.write_policy(&name, &policy::render_agent_with_queue(agent, hive)?)
|
||||
.await
|
||||
.with_context(|| format!("writing the read policy {name}"))?;
|
||||
store
|
||||
.write_cert_role(DEFAULT_CERT_MOUNT, &name, &authority.ca_pem, &name, &name)
|
||||
.await
|
||||
.with_context(|| format!("writing the cert-auth role {name}"))?;
|
||||
tracing::info!(agent, %path, role = %name, "agent store identity published");
|
||||
|
||||
read_back_as_agent(&credential, &name, &path).await?;
|
||||
tracing::info!(
|
||||
agent,
|
||||
role = %name,
|
||||
"agent store identity verified: the minted leaf logged in and read its own path"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The consumer of everything [`mint_and_verify`] wrote: log in **as the
|
||||
/// agent**, with the leaf just minted, and read back the path just published.
|
||||
///
|
||||
/// The address and the store's CA come from this process's own `BAO_*`
|
||||
/// environment; the *identity* deliberately does not — see
|
||||
/// [`SecretStore::connect_with_identity`]. The freshly minted private key
|
||||
/// never touches a filesystem.
|
||||
///
|
||||
/// # Errors
|
||||
/// When the store refuses the login (the role, the authority or the common
|
||||
/// name disagree), when the read is denied (the policy does not cover the
|
||||
/// path, or the role attached the wrong policy), or when what comes back is
|
||||
/// not what went in.
|
||||
async fn read_back_as_agent(credential: &mtls::Credential, role: &str, path: &str) -> Result<()> {
|
||||
let settings = Settings::from_env()
|
||||
.context("reading this daemon's own store settings for the read-back")?;
|
||||
|
||||
// The concatenated blob `connect_with_identity` takes, built in memory.
|
||||
let mut identity = credential.cert.clone().into_bytes();
|
||||
identity.push(b'\n');
|
||||
identity.extend_from_slice(credential.key.as_bytes());
|
||||
|
||||
let as_agent =
|
||||
SecretStore::connect_with_identity(&settings, &identity, role, DEFAULT_CERT_MOUNT)
|
||||
.await
|
||||
.with_context(|| {
|
||||
format!("logging in to the store as {role} with the leaf just minted")
|
||||
})?;
|
||||
let read_back: mtls::Credential = as_agent
|
||||
.read(path)
|
||||
.await
|
||||
.with_context(|| format!("reading {path} back under {role}'s own token"))?;
|
||||
|
||||
// Compared, not merely decoded: a successful read of an object written by
|
||||
// some earlier run would otherwise pass this check while this run's leaf
|
||||
// was the one nobody could use. Nothing about either value is printed.
|
||||
if read_back != *credential {
|
||||
bail!("the store returned a different object at {path} than the one just published");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{Authority, CLOCK_SKEW, LEAF_LIFETIME, validity};
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
/// A throwaway CA, minted in-process so no test needs a fixture file.
|
||||
fn test_authority() -> Authority {
|
||||
let key = rcgen::KeyPair::generate().expect("a key generates");
|
||||
let mut params = rcgen::CertificateParams::default();
|
||||
params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Constrained(0));
|
||||
params
|
||||
.distinguished_name
|
||||
.push(rcgen::DnType::CommonName, "swarm-agent-ca");
|
||||
let ca = params.self_signed(&key).expect("the CA self-signs");
|
||||
Authority::from_pem(ca.pem(), key.serialize_pem())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_window_is_backdated_by_the_skew_and_as_long_as_the_lifetime() {
|
||||
// The arithmetic that is wrong by a factor of sixty if a unit slips.
|
||||
let now = UNIX_EPOCH + Duration::from_secs(1_700_000_000);
|
||||
let (before, after) = validity(now, LEAF_LIFETIME).expect("a plain instant is fine");
|
||||
assert_eq!(before.unix_timestamp(), 1_700_000_000 - 300);
|
||||
assert_eq!(after.unix_timestamp(), 1_700_000_000 + 90 * 24 * 60 * 60);
|
||||
assert_eq!(CLOCK_SKEW, Duration::from_mins(5));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_minted_leaf_starts_valid_and_expires() {
|
||||
let now = i64::try_from(
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("the test host's clock is after 1970")
|
||||
.as_secs(),
|
||||
)
|
||||
.expect("and before the end of time");
|
||||
let (before, after) = validity(SystemTime::now(), LEAF_LIFETIME).expect("now is fine");
|
||||
assert!(
|
||||
before.unix_timestamp() < now,
|
||||
"a leaf usable only in the future is unusable"
|
||||
);
|
||||
assert!(
|
||||
after.unix_timestamp() > now,
|
||||
"a leaf that has already expired authenticates nothing"
|
||||
);
|
||||
// The rule `docs/swarm/credentials.md` states: no credential's
|
||||
// renewal strategy may read NONE, which starts with it having an end.
|
||||
// A decade-long leaf is that rule broken in a way review would miss.
|
||||
assert!(after.unix_timestamp() - now < 3653 * 24 * 60 * 60);
|
||||
}
|
||||
|
||||
/// The four-strings agreement `mint_and_verify` rests on, checked on the
|
||||
/// one of the four this process controls directly: the certificate really
|
||||
/// does carry the common name the cert-auth role will be told to match.
|
||||
#[test]
|
||||
fn the_leaf_carries_the_agents_object_name_as_its_common_name() {
|
||||
let name = swarm_secret_client::policy::agent_object_name("atlas").expect("legal");
|
||||
assert_eq!(name, "hive-agent-atlas");
|
||||
|
||||
let (cert, key) = test_authority().mint_leaf(&name).expect("the leaf signs");
|
||||
assert!(cert.contains("BEGIN CERTIFICATE"), "a PEM certificate");
|
||||
assert!(key.contains("PRIVATE KEY"), "a PEM key");
|
||||
|
||||
// Searched in the SIGNED DER rather than asserted on the params we
|
||||
// built: the claim is that the name reached the bytes a verifier
|
||||
// reads. A byte search rather than an X.509 parse because the whole
|
||||
// crate would otherwise gain a parser dependency for one assertion —
|
||||
// the name is a UTF8String in the subject DN, so it appears verbatim.
|
||||
let body: String = cert
|
||||
.lines()
|
||||
.filter(|l| !l.starts_with("-----"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("");
|
||||
let der = base64::Engine::decode(&base64::engine::general_purpose::STANDARD, body)
|
||||
.expect("the PEM body is base64");
|
||||
assert!(
|
||||
der.windows(name.len()).any(|w| w == name.as_bytes()),
|
||||
"the common name must be inside the signed certificate"
|
||||
);
|
||||
|
||||
// And the pair is a usable client identity — the first thing
|
||||
// `read_back_as_agent` does with it, in exactly this shape.
|
||||
let mut identity = cert.into_bytes();
|
||||
identity.push(b'\n');
|
||||
identity.extend_from_slice(key.as_bytes());
|
||||
reqwest::Identity::from_pem(&identity).expect("the leaf and its key form a TLS identity");
|
||||
}
|
||||
|
||||
/// A misconfiguration that would otherwise look like "not configured":
|
||||
/// half an authority has to be an error, not a silent `None`.
|
||||
///
|
||||
/// SAFETY: single-threaded mutation of two env vars no other test in this
|
||||
/// crate reads, removed again before returning.
|
||||
#[test]
|
||||
fn half_an_authority_is_an_error_and_neither_half_is_absence() {
|
||||
unsafe {
|
||||
std::env::remove_var(super::ENV_AGENT_CA);
|
||||
std::env::remove_var(super::ENV_AGENT_CA_KEY);
|
||||
}
|
||||
assert!(
|
||||
Authority::from_env()
|
||||
.expect("neither set is a supported shape")
|
||||
.is_none(),
|
||||
"a controller with no authority configured is not an error"
|
||||
);
|
||||
|
||||
unsafe { std::env::set_var(super::ENV_AGENT_CA, "/nonexistent/ca.pem") }
|
||||
// `.err().expect(..)` rather than `expect_err`: the `Ok` half is an
|
||||
// `Authority`, which deliberately has no `Debug` (it holds a key).
|
||||
let e = Authority::from_env()
|
||||
.err()
|
||||
.expect("a certificate with no key is half an authority");
|
||||
assert!(format!("{e:#}").contains(super::ENV_AGENT_CA_KEY), "{e:#}");
|
||||
|
||||
unsafe {
|
||||
std::env::remove_var(super::ENV_AGENT_CA);
|
||||
std::env::set_var(super::ENV_AGENT_CA_KEY, "/nonexistent/ca-key.pem");
|
||||
}
|
||||
// `.err().expect(..)` rather than `expect_err`: the `Ok` half is an
|
||||
// `Authority`, which deliberately has no `Debug` (it holds a key).
|
||||
let e = Authority::from_env()
|
||||
.err()
|
||||
.expect("a key with no certificate is the other half");
|
||||
assert!(format!("{e:#}").contains(super::ENV_AGENT_CA), "{e:#}");
|
||||
|
||||
unsafe { std::env::remove_var(super::ENV_AGENT_CA_KEY) }
|
||||
}
|
||||
}
|
||||
|
|
@ -40,6 +40,7 @@ use swarm_authelia_bridge_sock::BridgeResponse;
|
|||
use utoipa::{OpenApi, ToSchema};
|
||||
use utoipa_axum::{router::OpenApiRouter, routes};
|
||||
|
||||
mod agent_identity;
|
||||
mod agent_state_stream;
|
||||
mod agent_status;
|
||||
mod auth;
|
||||
|
|
@ -94,6 +95,16 @@ enum SwarmNodeKind {
|
|||
/// swarm routes a deploy message to — it belongs on the node that
|
||||
/// sends that message, not on this one.
|
||||
InitAgentConfigRepo { agent: String },
|
||||
/// Mint `agent`'s own client certificate for the swarm secret store,
|
||||
/// publish it there, grant it, and log in with it. See
|
||||
/// `agent_identity::mint_and_verify` — including why this node does not
|
||||
/// report success on a write.
|
||||
///
|
||||
/// Carries the hive for a different reason than `TriggerDeploy` does:
|
||||
/// not as an address, but because an agent's ACL document grants read on
|
||||
/// its hive's shared queue credential, so the document cannot be rendered
|
||||
/// without knowing which hive the agent belongs to.
|
||||
MintAgentIdentity { hive: String, agent: String },
|
||||
/// Tell `hive` to rebuild `agent`, by publishing on the swarm's deploy
|
||||
/// subject. The one node kind whose effect leaves this host.
|
||||
///
|
||||
|
|
@ -112,6 +123,7 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind {
|
|||
SwarmNodeKind::CreateForgeUser { .. } => "create_forge_user".to_owned(),
|
||||
SwarmNodeKind::AddRepoMember { .. } => "add_repo_member".to_owned(),
|
||||
SwarmNodeKind::InitAgentConfigRepo { .. } => "init_agent_config_repo".to_owned(),
|
||||
SwarmNodeKind::MintAgentIdentity { .. } => "mint_agent_identity".to_owned(),
|
||||
SwarmNodeKind::TriggerDeploy { .. } => "trigger_deploy".to_owned(),
|
||||
}
|
||||
}
|
||||
|
|
@ -131,7 +143,8 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind {
|
|||
| SwarmNodeKind::InitAgentConfigRepo { agent } => {
|
||||
serde_json::json!({ "agent": agent })
|
||||
}
|
||||
SwarmNodeKind::TriggerDeploy { hive, agent } => {
|
||||
SwarmNodeKind::TriggerDeploy { hive, agent }
|
||||
| SwarmNodeKind::MintAgentIdentity { hive, agent } => {
|
||||
serde_json::json!({ "agent": agent, "hive": hive })
|
||||
}
|
||||
}
|
||||
|
|
@ -168,6 +181,11 @@ struct WorkerDeps {
|
|||
/// connection living there is an accident of construction order, not a
|
||||
/// claim that events are a kind of status.
|
||||
queue: Option<async_nats::Client>,
|
||||
/// The authority agent client leaves are issued from, loaded once at
|
||||
/// startup because it holds a private key and a per-node re-read would be
|
||||
/// a per-node chance to read one. `None` on a host the operator has not
|
||||
/// given an authority — see `agent_identity::Authority::from_env`.
|
||||
agent_ca: Option<std::sync::Arc<agent_identity::Authority>>,
|
||||
}
|
||||
|
||||
/// Run a claimed node's actual work. Mirrors `hive-c0re/src/job_queue/
|
||||
|
|
@ -254,6 +272,20 @@ async fn run_swarm_node(
|
|||
Err(e) => Outcome::Failed(format!("{e:#}")),
|
||||
},
|
||||
},
|
||||
SwarmNodeKind::MintAgentIdentity { hive, agent } => match deps.agent_ca {
|
||||
None => Outcome::Failed(
|
||||
"no agent certificate authority is configured on this host \
|
||||
(SWARM_CONTROLLER_AGENT_CA_FILE / SWARM_CONTROLLER_AGENT_CA_KEY_FILE unset), \
|
||||
so this agent has no identity at the swarm secret store"
|
||||
.to_owned(),
|
||||
),
|
||||
Some(authority) => {
|
||||
match agent_identity::mint_and_verify(&authority, &agent, &hive).await {
|
||||
Ok(()) => Outcome::Done,
|
||||
Err(e) => Outcome::Failed(format!("{e:#}")),
|
||||
}
|
||||
}
|
||||
},
|
||||
SwarmNodeKind::TriggerDeploy { hive, agent } => match deps.queue {
|
||||
None => Outcome::Failed(
|
||||
"no swarm queue is configured on this host, so no hive can be told to deploy"
|
||||
|
|
@ -1194,50 +1226,7 @@ async fn create_agent(
|
|||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let ids = sched
|
||||
.insert_job(None, |b| {
|
||||
let create_identity = b.node(SwarmNodeKind::CreateIdentity {
|
||||
agent: agent.clone(),
|
||||
});
|
||||
// A second, independent root: a forge user needs neither an
|
||||
// authelia subject nor an existing repo, so it does not chain
|
||||
// off `create_identity` (see the doc comment above).
|
||||
let create_forge_user = b.node(SwarmNodeKind::CreateForgeUser {
|
||||
agent: agent.clone(),
|
||||
});
|
||||
let create_repo = b
|
||||
.node(SwarmNodeKind::CreateRepo {
|
||||
agent: agent.clone(),
|
||||
})
|
||||
.after_ok(create_identity);
|
||||
// `AddRepoMember` needs both parents: the repo to add a
|
||||
// collaborator to, and the forge user to add as one — adding a
|
||||
// nonexistent user is a Forgejo validation error, not
|
||||
// an idempotent no-op. `InitAgentConfigRepo` needs only the
|
||||
// repo — see the doc comment above for why.
|
||||
let _add_repo_member = b
|
||||
.node(SwarmNodeKind::AddRepoMember {
|
||||
agent: agent.clone(),
|
||||
})
|
||||
.after_ok(create_repo)
|
||||
.after_ok(create_forge_user);
|
||||
let init_config = b
|
||||
.node(SwarmNodeKind::InitAgentConfigRepo {
|
||||
agent: agent.clone(),
|
||||
})
|
||||
.after_ok(create_repo);
|
||||
// Last, and specifically after the config repo is seeded: the
|
||||
// hive deploys by reading that repo, so a deploy asked for any
|
||||
// earlier would find nothing to build. This is the edge that
|
||||
// makes creating an agent at swarm level actually put it on a
|
||||
// hive, rather than leaving a provisioned name nobody runs.
|
||||
let _trigger_deploy = b
|
||||
.node(SwarmNodeKind::TriggerDeploy {
|
||||
hive: hive.clone(),
|
||||
agent,
|
||||
})
|
||||
.after_ok(init_config);
|
||||
vec![create_identity.guid()]
|
||||
})
|
||||
.insert_job(None, |b| declare_agent_job(b, &agent, &hive))
|
||||
.map_err(|e| {
|
||||
error_problem(
|
||||
axum::http::StatusCode::INTERNAL_SERVER_ERROR,
|
||||
|
|
@ -1253,6 +1242,101 @@ async fn create_agent(
|
|||
}))
|
||||
}
|
||||
|
||||
/// The authority agent leaves are issued from, or `None` on a host that was
|
||||
/// given none.
|
||||
///
|
||||
/// Same "log and carry on" shape as `main`'s other optional wiring: a
|
||||
/// controller with no agent authority still serves everything else, and
|
||||
/// `MintAgentIdentity` fails with a named reason rather than this process
|
||||
/// refusing to start. The `Err` arm is worth its own warning — half an
|
||||
/// authority, or a file that will not read, is a host that looks configured
|
||||
/// and mints nothing.
|
||||
fn load_agent_authority() -> Option<Arc<agent_identity::Authority>> {
|
||||
match agent_identity::Authority::from_env() {
|
||||
Ok(authority) => authority.map(Arc::new),
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
error = %format!("{e:#}"),
|
||||
"agent certificate authority unusable; agents get no store identity here"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The sub-DAG one agent creation is: the nodes, and the edges between them.
|
||||
///
|
||||
/// A function rather than a closure inside [`create_agent`] so the endpoint's
|
||||
/// validation and the graph's shape can each be read without scrolling past
|
||||
/// the other — and so the one handle the response reports is returned from
|
||||
/// the place that decides which node it is.
|
||||
fn declare_agent_job(
|
||||
b: &hive_jobq::builder::JobBuilder<SwarmNodeKind, SwarmResourceKind>,
|
||||
agent: &str,
|
||||
hive: &str,
|
||||
) -> Vec<hive_jobq::builder::NodeGuid> {
|
||||
let create_identity = b.node(SwarmNodeKind::CreateIdentity {
|
||||
agent: agent.to_owned(),
|
||||
});
|
||||
// A second, independent root: a forge user needs neither an authelia
|
||||
// subject nor an existing repo, so it does not chain off
|
||||
// `create_identity` (see the doc comment above).
|
||||
let create_forge_user = b.node(SwarmNodeKind::CreateForgeUser {
|
||||
agent: agent.to_owned(),
|
||||
});
|
||||
let create_repo = b
|
||||
.node(SwarmNodeKind::CreateRepo {
|
||||
agent: agent.to_owned(),
|
||||
})
|
||||
.after_ok(create_identity);
|
||||
// `AddRepoMember` needs both parents: the repo to add a collaborator to,
|
||||
// and the forge user to add as one — adding a nonexistent user is a
|
||||
// Forgejo validation error, not an idempotent no-op.
|
||||
// `InitAgentConfigRepo` needs only the repo — see the doc comment above
|
||||
// for why.
|
||||
let _add_repo_member = b
|
||||
.node(SwarmNodeKind::AddRepoMember {
|
||||
agent: agent.to_owned(),
|
||||
})
|
||||
.after_ok(create_repo)
|
||||
.after_ok(create_forge_user);
|
||||
let init_config = b
|
||||
.node(SwarmNodeKind::InitAgentConfigRepo {
|
||||
agent: agent.to_owned(),
|
||||
})
|
||||
.after_ok(create_repo);
|
||||
// The agent's own identity at the swarm secret store, minted and
|
||||
// published at swarm level so no hive ever needs the capability to mint
|
||||
// one. After `create_identity` because the certificate names a principal
|
||||
// the swarm has agreed exists — not because anything in the store reads
|
||||
// authelia.
|
||||
let mint_identity = b
|
||||
.node(SwarmNodeKind::MintAgentIdentity {
|
||||
hive: hive.to_owned(),
|
||||
agent: agent.to_owned(),
|
||||
})
|
||||
.after_ok(create_identity);
|
||||
// Last, and specifically after the config repo is seeded: the hive
|
||||
// deploys by reading that repo, so a deploy asked for any earlier would
|
||||
// find nothing to build. This is the edge that makes creating an agent at
|
||||
// swarm level actually put it on a hive, rather than leaving a
|
||||
// provisioned name nobody runs.
|
||||
//
|
||||
// `after_any` on the mint, not `after_ok`: a hive cannot pass down a
|
||||
// certificate the swarm has not published, so the deploy must not
|
||||
// overtake the mint — but a host with no authority configured must still
|
||||
// create agents exactly as it does today. `after_ok` there would turn an
|
||||
// unconfigured option into an agent nobody runs.
|
||||
let _trigger_deploy = b
|
||||
.node(SwarmNodeKind::TriggerDeploy {
|
||||
hive: hive.to_owned(),
|
||||
agent: agent.to_owned(),
|
||||
})
|
||||
.after_ok(init_config)
|
||||
.after_any(mint_identity);
|
||||
vec![create_identity.guid()]
|
||||
}
|
||||
|
||||
/// Query params for `GET /api/jobq/graph` — `?states=` narrows to root
|
||||
/// groups in the named states, same shape `hive_jobq_wire::parse_states`
|
||||
/// parses.
|
||||
|
|
@ -1579,6 +1663,7 @@ async fn main() -> Result<()> {
|
|||
auth: auth.clone(),
|
||||
forge: forge_client.clone(),
|
||||
queue: status.as_ref().map(|s| s.queue_client()),
|
||||
agent_ca: load_agent_authority(),
|
||||
};
|
||||
|
||||
let jobq = Arc::new(Mutex::new(hive_jobq::scheduler::Scheduler::new(
|
||||
|
|
@ -1945,6 +2030,7 @@ mod tests {
|
|||
auth: None,
|
||||
forge: None,
|
||||
queue: None,
|
||||
agent_ca: None,
|
||||
};
|
||||
let runner =
|
||||
hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| {
|
||||
|
|
@ -1997,6 +2083,7 @@ mod tests {
|
|||
auth: None,
|
||||
forge: None,
|
||||
queue: None,
|
||||
agent_ca: None,
|
||||
};
|
||||
let runner =
|
||||
hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| {
|
||||
|
|
@ -2018,6 +2105,130 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
/// Third sibling of the two above, and the same deliberate caveat: with
|
||||
/// no authority configured this reaches only the
|
||||
/// graceful-absence-is-failure branch. The happy path is a live store
|
||||
/// and a real login, which is exactly why it is `mint_and_verify`'s own
|
||||
/// job to prove it at agent-creation time rather than a unit test's.
|
||||
///
|
||||
/// What this does pin is the degrade: a host that was never given an
|
||||
/// authority fails this one node with a reason that names the two
|
||||
/// variables, and creates the agent anyway.
|
||||
#[tokio::test]
|
||||
async fn mint_agent_identity_node_runs_end_to_end_and_fails_without_an_authority() {
|
||||
let mut sched = hive_jobq::scheduler::Scheduler::new(
|
||||
hive_jobq::Graph::new(),
|
||||
hive_jobq::resources::ResourceTable::new(),
|
||||
);
|
||||
let id = sched
|
||||
.append(
|
||||
SwarmNodeKind::MintAgentIdentity {
|
||||
hive: "pr1ma".to_owned(),
|
||||
agent: "atlas".to_owned(),
|
||||
},
|
||||
Vec::new(),
|
||||
None,
|
||||
)
|
||||
.expect("insert");
|
||||
let sched = std::sync::Arc::new(std::sync::Mutex::new(sched));
|
||||
|
||||
let deps = WorkerDeps {
|
||||
auth: None,
|
||||
forge: None,
|
||||
queue: None,
|
||||
agent_ca: None,
|
||||
};
|
||||
let runner =
|
||||
hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| {
|
||||
run_swarm_node(id, kind, builder, deps)
|
||||
})
|
||||
.expect("the node just inserted is runnable");
|
||||
runner
|
||||
.await
|
||||
.1
|
||||
.expect("no growth declared, nothing to reject");
|
||||
|
||||
let guard = sched.lock().unwrap();
|
||||
let node = guard.graph().node(id).expect("node still present");
|
||||
assert_eq!(node.state, hive_jobq::State::Failed);
|
||||
let error = node.error.as_deref().unwrap_or_default();
|
||||
assert!(
|
||||
error.contains(crate::agent_identity::ENV_AGENT_CA)
|
||||
&& error.contains(crate::agent_identity::ENV_AGENT_CA_KEY),
|
||||
"the reason must name both variables an operator has to set, got {error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// The node carries the hive, and the viewer has to see it. The `data`
|
||||
/// match is an or-pattern on purpose (see its own comment), and this is
|
||||
/// the assertion that the new variant joined the two-field arm rather
|
||||
/// than the agent-only one — a viewer silently missing the hive is the
|
||||
/// failure that comment describes having already happened once.
|
||||
#[test]
|
||||
fn a_mint_node_renders_both_the_agent_and_the_hive() {
|
||||
use hive_jobq_wire::WireNode as _;
|
||||
|
||||
let kind = SwarmNodeKind::MintAgentIdentity {
|
||||
hive: "pr1ma".to_owned(),
|
||||
agent: "atlas".to_owned(),
|
||||
};
|
||||
assert_eq!(kind.label(), "mint_agent_identity");
|
||||
let data = kind.data(1);
|
||||
assert_eq!(data["agent"], "atlas");
|
||||
assert_eq!(data["hive"], "pr1ma");
|
||||
}
|
||||
|
||||
/// The ordering the operator's ruling requires: a hive cannot pass down
|
||||
/// a certificate the swarm has not published, so the deploy message must
|
||||
/// not leave before the mint is terminal.
|
||||
///
|
||||
/// Asserted on the graph `create_agent` builds, because the edge is one
|
||||
/// line in a builder closure and its absence changes nothing observable
|
||||
/// until a real agent boots without an identity.
|
||||
#[tokio::test]
|
||||
async fn the_deploy_waits_for_the_mint_and_is_not_cancelled_by_it() {
|
||||
use hive_jobq_wire::WireNode as _;
|
||||
|
||||
let (state, sched) = state_with_roster();
|
||||
let _queued = super::create_agent(
|
||||
axum::extract::State(state),
|
||||
axum::Json(super::CreateAgentRequest {
|
||||
name: "atlas".to_owned(),
|
||||
hive: "pr1ma".to_owned(),
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("a hive in the roster must be accepted");
|
||||
|
||||
let guard = sched
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let graph = guard.graph();
|
||||
let id_of = |label: &str| {
|
||||
graph
|
||||
.nodes()
|
||||
.find(|n| n.payload.label() == label)
|
||||
.unwrap_or_else(|| panic!("the graph holds a {label} node"))
|
||||
.id
|
||||
};
|
||||
let mint = id_of("mint_agent_identity");
|
||||
let deploy = graph.node(id_of("trigger_deploy")).expect("just found");
|
||||
|
||||
let when = deploy
|
||||
.deps
|
||||
.iter()
|
||||
.find_map(|d| match d {
|
||||
hive_jobq::Dep::Node { id, when } if *id == mint => Some(*when),
|
||||
_ => None,
|
||||
})
|
||||
.expect("the deploy waits for the mint");
|
||||
assert!(
|
||||
when.accepts(hive_jobq::TerminalState::Failed),
|
||||
"an unconfigured authority must not cancel the deploy; this edge \
|
||||
has to be `after_any`, not `after_ok`"
|
||||
);
|
||||
}
|
||||
|
||||
/// The socket must not share a directory with anything else, because
|
||||
/// the socket is `0666` and the directory is therefore the only access
|
||||
/// control it has. `/run/hyperhive` in particular holds hive-c0re's
|
||||
|
|
|
|||
|
|
@ -112,7 +112,37 @@ impl SecretStore {
|
|||
let mut pem = read_file(ENV_CLIENT_CERT, &settings.cert_path)?;
|
||||
pem.push(b'\n');
|
||||
pem.extend_from_slice(&read_file(ENV_CLIENT_KEY, &settings.key_path)?);
|
||||
let identity = reqwest::Identity::from_pem(&pem).map_err(Error::Tls)?;
|
||||
Self::connect_with_identity(settings, &pem, cert_role, cert_mount).await
|
||||
}
|
||||
|
||||
/// Log in at the store `settings` names, presenting an identity the caller
|
||||
/// is already holding rather than one named by the environment.
|
||||
///
|
||||
/// `identity_pem` is the certificate and its private key concatenated into
|
||||
/// one PEM blob — the shape [`reqwest::Identity::from_pem`] takes, and the
|
||||
/// shape [`SecretStore::connect`] builds out of the two files
|
||||
/// [`ENV_CLIENT_CERT`] and [`ENV_CLIENT_KEY`] name. Only the address and
|
||||
/// the CA are read out of `settings` here; its two identity paths are not
|
||||
/// touched.
|
||||
///
|
||||
/// This is what a **minter** needs. A process that has just issued a leaf
|
||||
/// holds the bytes, and the safest place for a freshly minted private key
|
||||
/// is the memory it was generated in: writing it to a file purely so that
|
||||
/// a `Settings` could name it would put a key on disk for no other reason
|
||||
/// than to log in with it once.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`Error::Tls`] when `identity_pem` is not a usable certificate/key
|
||||
/// pair, [`Error::Settings`] when the address will not parse, and
|
||||
/// [`Error::Vault`] when the store refuses the login — which is what a
|
||||
/// certificate no cert-auth role accepts looks like from here.
|
||||
pub async fn connect_with_identity(
|
||||
settings: &Settings,
|
||||
identity_pem: &[u8],
|
||||
cert_role: &str,
|
||||
cert_mount: &str,
|
||||
) -> Result<Self, Error> {
|
||||
let identity = reqwest::Identity::from_pem(identity_pem).map_err(Error::Tls)?;
|
||||
|
||||
let mut builder = VaultClientSettingsBuilder::default();
|
||||
builder
|
||||
|
|
|
|||
|
|
@ -5,8 +5,8 @@
|
|||
//! the rules every path obeys ([`path`]), the translation from this
|
||||
//! deployment's environment into a logged-in client ([`client`]), and, per kind
|
||||
//! of secret, the path it lives at together with the fields it holds
|
||||
//! ([`matrix`], [`queue`]). Each of those is a thing the controller and a hive
|
||||
//! must say identically, so it is said once here.
|
||||
//! ([`matrix`], [`queue`], [`mtls`]). Each of those is a thing the controller
|
||||
//! and a hive must say identically, so it is said once here.
|
||||
//!
|
||||
//! [`policy`] is the same kind of agreement seen from the other side: which of
|
||||
//! those paths a given principal's own token may read — a hive's, and an
|
||||
|
|
@ -18,9 +18,13 @@
|
|||
//! [`client`] is deliberately ignorant of all of it: it moves whatever type a
|
||||
//! caller names, so a second kind of secret is a new module beside [`matrix`]
|
||||
//! and not another field on a struct shared with it.
|
||||
//!
|
||||
//! [`mtls`] is the one module about reaching the store rather than about a
|
||||
//! value inside it, and its doc explains why that is not circular.
|
||||
|
||||
pub mod client;
|
||||
pub mod matrix;
|
||||
pub mod mtls;
|
||||
pub mod path;
|
||||
pub mod policy;
|
||||
pub mod queue;
|
||||
|
|
|
|||
173
swarm-secret-client/src/mtls.rs
Normal file
173
swarm-secret-client/src/mtls.rs
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
//! The mTLS agreement: where an agent's own identity **at the store** lives,
|
||||
//! and what the object at that path holds.
|
||||
//!
|
||||
//! The sibling of [`crate::matrix`] and [`crate::queue`], and the one that is
|
||||
//! about reaching the store rather than about something kept inside it. An
|
||||
//! agent's client certificate is minted at swarm level
|
||||
//! (`swarm-controller::agent_identity`) and published here; the agent's hive
|
||||
//! collects it under the **hive's** own certificate and hands it into the
|
||||
//! container.
|
||||
//!
|
||||
//! 🔑 The recursion this shape looks like it has — *a credential fetched from
|
||||
//! the store that is what opens the store* — is broken by who reads it. The
|
||||
//! reader is the hive, never the agent, and a hive already holds its own leaf.
|
||||
//! Nothing has to already be an agent in order to obtain an agent's identity.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{
|
||||
Error,
|
||||
path::{Kind, principal_prefix},
|
||||
};
|
||||
|
||||
/// The path holding `agent`'s client certificate for the store itself.
|
||||
///
|
||||
/// One path per agent with no further segment below it, unlike
|
||||
/// [`crate::matrix::account_path`]: an agent has exactly one identity, and a
|
||||
/// second one under a name would be a second principal wearing that name.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`Error::PathSegment`] when `agent` contains anything but `[A-Za-z0-9_-]`,
|
||||
/// which is what keeps one agent's name from addressing another's identity.
|
||||
pub fn identity_path(agent: &str) -> Result<String, Error> {
|
||||
let prefix = principal_prefix(Kind::Agent, agent)?;
|
||||
Ok(format!("{prefix}/bao-mtls"))
|
||||
}
|
||||
|
||||
/// What the path holds: the leaf, its private key, and the authority the leaf
|
||||
/// was issued from.
|
||||
///
|
||||
/// All three, because a reader has to reconstruct a usable identity from the
|
||||
/// store alone — the same requirement [`crate::queue::Credential`] states for
|
||||
/// carrying its client id. The authority rides along so the cert-auth role
|
||||
/// and the certificate cannot be delivered from two different sources and
|
||||
/// silently disagree; it is public material, unlike the other two fields.
|
||||
///
|
||||
/// ⚠️ **No `Debug` derive.** See the hand-written impl below: one of these
|
||||
/// fields is a private key, and a derived `Debug` would put it in any log line
|
||||
/// that ever formatted a node's payload.
|
||||
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct Credential {
|
||||
/// The agent's client certificate, PEM. The nix-side reader in an agent's
|
||||
/// hive spells `bao kv get -field=cert`, so this name is load-bearing for
|
||||
/// a reader this crate does not control.
|
||||
pub cert: String,
|
||||
|
||||
/// The private key for [`Credential::cert`], PEM. `bao kv get -field=key`
|
||||
/// on the reading side, and `0600` the moment it lands on disk there.
|
||||
pub key: String,
|
||||
|
||||
/// The authority [`Credential::cert`] was issued from, PEM. Public
|
||||
/// material: it is also what the store keeps inside the agent's cert-auth
|
||||
/// role, by value (see [`crate::client::SecretStore::write_cert_role`]).
|
||||
pub ca: String,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for Credential {
|
||||
/// Redacts the key and summarises the two public fields by length.
|
||||
///
|
||||
/// Hand-written rather than derived because the derive is the failure:
|
||||
/// this type is carried through a job-graph node and an `anyhow` context
|
||||
/// chain, both of which format whatever they are given.
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("Credential")
|
||||
.field("cert", &format_args!("{} bytes of PEM", self.cert.len()))
|
||||
.field("key", &"<redacted>")
|
||||
.field("ca", &format_args!("{} bytes of PEM", self.ca.len()))
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn an_agent_name_lands_under_its_own_principal_prefix() {
|
||||
assert_eq!(
|
||||
identity_path("atlas").expect("a plain name is legal"),
|
||||
"swarm/agents/atlas/bao-mtls"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_traversal_in_the_agent_name_is_refused() {
|
||||
let e = identity_path("../argus").expect_err("a traversal is not");
|
||||
assert!(matches!(e, Error::PathSegment { kind: "agent", .. }), "{e}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_agents_never_share_a_path() {
|
||||
assert_ne!(
|
||||
identity_path("atlas").expect("legal"),
|
||||
identity_path("argus").expect("legal")
|
||||
);
|
||||
}
|
||||
|
||||
/// The path sits under the same prefix an agent's policy grants
|
||||
/// (`policy::render_agent`'s stanza is `swarm/agents/<agent>/*`), which is
|
||||
/// what makes an agent able to read its own identity back.
|
||||
#[test]
|
||||
fn an_agents_identity_is_inside_its_own_policy_stanza() {
|
||||
let path = identity_path("atlas").expect("legal");
|
||||
let document = crate::policy::render_agent("atlas").expect("legal");
|
||||
let prefix = format!("{}/data/swarm/agents/atlas/", crate::path::MOUNT);
|
||||
assert!(
|
||||
path.starts_with("swarm/agents/atlas/"),
|
||||
"the identity must sit under the agent's own prefix, got {path}"
|
||||
);
|
||||
assert!(
|
||||
document.contains(&format!("{prefix}*")),
|
||||
"the agent's document must cover {prefix}*, got:\n{document}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_object_round_trips_through_the_store_representation() {
|
||||
let c = Credential {
|
||||
cert: "-----BEGIN CERTIFICATE-----\n".to_owned(),
|
||||
key: "-----BEGIN PRIVATE KEY-----\n".to_owned(),
|
||||
ca: "-----BEGIN CERTIFICATE-----\n".to_owned(),
|
||||
};
|
||||
let json = serde_json::to_string(&c).expect("serialises");
|
||||
assert_eq!(
|
||||
serde_json::from_str::<Credential>(&json).expect("deserialises"),
|
||||
c
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_field_names_the_nix_reader_asks_for_are_the_ones_written() {
|
||||
let json = serde_json::to_value(Credential {
|
||||
cert: "leaf".to_owned(),
|
||||
key: "private".to_owned(),
|
||||
ca: "authority".to_owned(),
|
||||
})
|
||||
.expect("serialises");
|
||||
assert_eq!(json["cert"], "leaf");
|
||||
assert_eq!(json["key"], "private");
|
||||
assert_eq!(json["ca"], "authority");
|
||||
}
|
||||
|
||||
/// The property the hand-written `Debug` exists for: a key that reaches a
|
||||
/// log line is a key on a disk somebody else owns.
|
||||
#[test]
|
||||
fn formatting_the_credential_does_not_reveal_the_key() {
|
||||
let rendered = format!(
|
||||
"{:?}",
|
||||
Credential {
|
||||
cert: "leaf".to_owned(),
|
||||
key: "SUPER-SECRET-KEY-MATERIAL".to_owned(),
|
||||
ca: "authority".to_owned(),
|
||||
}
|
||||
);
|
||||
assert!(
|
||||
!rendered.contains("SUPER-SECRET-KEY-MATERIAL"),
|
||||
"the key must not survive formatting, got {rendered}"
|
||||
);
|
||||
assert!(
|
||||
rendered.contains("<redacted>"),
|
||||
"and the reader must be told it was withheld, got {rendered}"
|
||||
);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue