diff --git a/Cargo.lock b/Cargo.lock index c694d29e..64c1b82b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -162,6 +162,45 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9dbc3a507a82b17ba0d98f6ce8fd6954ea0c8152e98009d36a40d8dcc8ce078a" +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure 0.13.2", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "assign" version = "1.1.1" @@ -198,7 +237,7 @@ version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d83a251fa1a4c9d0fe6e816b7acd60549e473e08d14f27a1d992c2675abff05f" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-util", "memchr", @@ -308,7 +347,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ "axum-core", - "base64", + "base64 0.22.1", "bytes", "form_urlencoded", "futures-util", @@ -373,6 +412,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64ct" version = "1.8.3" @@ -385,13 +430,22 @@ version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f3c067aa24dd4ed5c79cf222a38f260c8f23d3b82a062fba3f28c6fe563b753" dependencies = [ - "base64", + "base64 0.22.1", "blowfish", "getrandom 0.4.3", "subtle", "zeroize", ] +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "2.13.1" @@ -1035,6 +1089,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "deranged" version = "0.5.8" @@ -1740,7 +1808,7 @@ dependencies = [ "anyhow", "async-nats", "axum", - "base64", + "base64 0.22.1", "bcrypt", "chrono", "clap", @@ -2190,7 +2258,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -2903,7 +2971,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fef37395fffb7c916f7109ab0d16d8ca599403dd8164d08c0d966176b66ede47" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "futures-util", "getrandom 0.4.3", "gloo-utils", @@ -2962,7 +3030,7 @@ version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f48f304e553fb6200b1d7d1f77a88fd182076d4b25624e9dbfa42d6a37de35e" dependencies = [ - "base64", + "base64 0.22.1", "blake3", "chacha20poly1305", "getrandom 0.2.17", @@ -3045,6 +3113,12 @@ dependencies = [ "unicase", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -3100,6 +3174,16 @@ dependencies = [ "signatory", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -3109,12 +3193,31 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -3140,7 +3243,7 @@ version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" dependencies = [ - "base64", + "base64 0.22.1", "chrono", "getrandom 0.2.17", "http", @@ -3163,6 +3266,15 @@ dependencies = [ "reqwest", ] +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -3237,7 +3349,7 @@ version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" dependencies = [ - "base64", + "base64 0.22.1", "const-hex", "opentelemetry", "opentelemetry_sdk", @@ -3311,6 +3423,16 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "pem" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" +dependencies = [ + "base64 0.23.1", + "serde_core", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -3754,6 +3876,20 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rcgen" +version = "0.14.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "x509-parser", + "yasna", +] + [[package]] name = "readlock" version = "0.1.11" @@ -3833,7 +3969,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-channel", @@ -3995,7 +4131,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c3b4f00112791b490acce57df1ce3eb3f88899b045bebcff8a29f75369640cc" dependencies = [ "as_variant", - "base64", + "base64 0.22.1", "bytes", "date_header", "form_urlencoded", @@ -4112,6 +4248,15 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + [[package]] name = "rustify" version = "0.7.0" @@ -4734,7 +4879,7 @@ dependencies = [ "async-nats", "async-trait", "axum", - "base64", + "base64 0.22.1", "bytes", "forgejo-api", "futures-util", @@ -4749,6 +4894,7 @@ dependencies = [ "opentelemetry-otlp", "opentelemetry_sdk", "problem_details", + "rcgen", "reqwest", "serde", "serde_json", @@ -4757,6 +4903,7 @@ dependencies = [ "swarm-authelia-bridge-sock", "swarm-queue-client", "swarm-secret-client", + "time", "tokio", "tracing", "tracing-subscriber", @@ -5137,7 +5284,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f591660438b3038dd04d16c938271c79e7e06260ad2ea2885a4861bfb238605d" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "futures-sink", @@ -5579,7 +5726,7 @@ checksum = "b98bf83c0992966775b8012f194b07b44928996163e5a05b741b43891571ae5b" dependencies = [ "aes", "arrayvec", - "base64", + "base64 0.22.1", "base64ct", "cbc", "chacha20poly1305", @@ -6069,12 +6216,40 @@ dependencies = [ "zeroize", ] +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "ring", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + [[package]] name = "xxhash-rust" version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "985eec839aaf2a1270af8f4ebcf63cf9401cfd90f0902f97c28d9f104ffbde72" +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" +dependencies = [ + "bit-vec", + "time", +] + [[package]] name = "yoke" version = "0.8.3" diff --git a/Cargo.toml b/Cargo.toml index 040b3501..a56fe0f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,6 +101,20 @@ vaultrs = "0.8" # resolves, since its `exec_with_empty` takes *its* `Endpoint` trait. rustify = "0.7" rustify_derive = "0.5" +# Signs the per-agent client leaf `swarm-controller::agent_identity` mints. +# A library rather than an `openssl` shellout, which is what every other CA +# in this tree is (`glue-bao-tls.nix`, `hive-tls.nix`, `swarm-ca.nix`): those +# run once at deploy time and write to disk, this one runs per agent +# creation and must keep the private key it generates in memory long enough +# to log in with it and no longer. `ring` over `aws_lc_rs` because `ring` is +# already in the lock; the crypto backend is not otherwise load-bearing. +# +# `x509-parser`: `Issuer::from_ca_cert_pem` is gated behind it, and reading +# the authority back out of its own PEM is how a leaf inherits the subject and +# key identifier that make it chain. The ungated constructors take a +# `CertificateParams` the caller would have to restate by hand — two spellings +# of one authority, agreeing until the day they don't. +rcgen = { version = "0.14", features = ["x509-parser"] } tower-http = { version = "0.7", features = ["fs"] } uuid = { version = "1", features = ["v4"] } rmcp = { version = "2", default-features = false, features = [ diff --git a/docs/swarm/credentials.md b/docs/swarm/credentials.md index b0a5c89e..7a7dd9cc 100644 --- a/docs/swarm/credentials.md +++ b/docs/swarm/credentials.md @@ -48,13 +48,24 @@ path at runtime — not a path on disk, and not a unit whose job is to turn a store value into a file. A renewal cell may never read `NONE`: state the strategy for every credential, including the mTLS leaf. -| store path | minter | reader — pulls at runtime, holds in memory | renewal | -| -------------------------------------------- | ------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------ | -------------- | -| `swarm/agents//matrix/` | `swarm-controller` | the agent container itself, under the certificate its hive passed in | must be stated | -| `swarm/hives//matrix/appservice-token` | one minter, on the authelia host | the hive process that presents the token to its homeserver, under the hive's own certificate | must be stated | -| `swarm/hives//queue/agent` | authelia | the agent container presenting the OIDC client to the swarm queue, under its own certificate | must be stated | -| `swarm/services//oidc/client` | authelia | the service process that presents the client secret, under the certificate of the host it runs on | must be stated | -| _(not in the store)_ the mTLS client leaf | the store's own PKI, or an operator placing it by hand | its own client, off disk — the sole exception above, because it's what makes every other row's pull possible | must be stated | +| store path | minter | reader — pulls at runtime, holds in memory | renewal | +| -------------------------------------------- | ------------------------------------------------------ | ------------------------------------------------------------------------------------------------------- | -------------- | +| `swarm/agents//matrix/` | `swarm-controller` | the agent container itself, under the certificate its hive passed in | must be stated | +| `swarm/agents//bao-mtls` | `swarm-controller`, at agent creation | the agent's **hive**, under the hive's own certificate, which hands it into the container | must be stated | +| `swarm/hives//matrix/appservice-token` | one minter, on the authelia host | the hive process that presents the token to its homeserver, under the hive's own certificate | must be stated | +| `swarm/hives//queue/agent` | authelia | the agent container presenting the OIDC client to the swarm queue, under its own certificate | must be stated | +| `swarm/services//oidc/client` | authelia | the service process that presents the client secret, under the certificate of the host it runs on | must be stated | +| _(not in the store)_ a hive's mTLS leaf | the store's own PKI, or an operator placing it by hand | its own client, off disk — the exception above, because it's what makes every other row's pull possible | must be stated | + +**An agent's mTLS leaf is in the store; a hive's is not, and the difference +is not an inconsistency.** The rule the exception protects is that nothing +can fetch from the store the credential it would need in order to fetch. A +hive's leaf is that credential, so it can only come off disk. An agent's is +read by the _hive_, which already holds one — so publishing it costs nothing +and buys the property this page asks for: the swarm mints it, the hive only +carries it, and no hive ever needs the capability to mint an identity. +`swarm-controller` proves the leaf it publishes before the creation job +reports success, by logging in with it and reading the row back. ## Progressive enhancement diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix index d3f12383..1a1e0932 100644 --- a/nix/host-modules/swarm-controller.nix +++ b/nix/host-modules/swarm-controller.nix @@ -37,6 +37,33 @@ let # file would be handed to a daemon that cannot use it. haveHiveClientCa = haveBaoIdentity && deployCfg.swarm-controller.hiveClientCaFile != null; + # The authority this daemon issues AGENT client leaves from — a different + # question from `hiveClientCaFile` above, which is the authority it *trusts* + # hives by. This one it signs with, so it needs the private key too. + # + # ⚠️ Deliberately NOT the store's own PKI (`glue-bao-tls.nix`'s + # `/var/lib/swarm-bao-pki`). A cert-auth role pins its authority by value, + # per role, so a role this daemon writes carries whatever authority this + # daemon hands it — which is what lets the controller mint from its own CA + # on its own host without anything being co-located and without any + # existing role changing. `swarm-controller/src/agent_identity.rs`'s module + # doc is the long form. + agentCaDir = "/var/lib/swarm-controller-agent-ca"; + + # No authority named means mint one here. The alternative — leaving agent + # identities off until an operator places a CA by hand — is the state where + # the whole path is configured and silently does nothing, which is the + # failure mode `glue-bao-tls.nix` avoids the same way. + selfSignAgentCa = deployCfg.swarm-controller.agentCaFile == null; + agentCaCert = + if selfSignAgentCa then "${agentCaDir}/ca.pem" else deployCfg.swarm-controller.agentCaFile; + agentCaKey = + if selfSignAgentCa then "${agentCaDir}/ca-key.pem" else deployCfg.swarm-controller.agentCaKeyFile; + + # Minting an agent's identity means publishing it to the store, so the + # authority alone is not enough — same rule `haveHiveClientCa` states. + haveAgentCa = haveBaoIdentity && agentCaKey != null; + # `swarm_secret_client` reads these spellings explicitly rather than # vaultrs's `VAULT_*` defaults — falling through to those builds a client # with no identity and fails at the TLS handshake, naming neither. `%d` and @@ -59,6 +86,13 @@ let # store should trust hives by, which the controller reads as a value # to put in each hive's cert-auth role. SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE = "%d/hive-client-ca.pem"; + } + // lib.optionalAttrs haveAgentCa { + # The authority agent leaves are ISSUED FROM, so unlike every other + # `*_CA_FILE` here it comes with a key. `%d` for both: the key is + # `0600` and root-owned, and this daemon runs unprivileged. + SWARM_CONTROLLER_AGENT_CA_FILE = "%d/agent-ca.pem"; + SWARM_CONTROLLER_AGENT_CA_KEY_FILE = "%d/agent-ca-key.pem"; }; # What `swarmctl` needs in order to act on authelia from the host. @@ -604,6 +638,47 @@ in ''; }; + agentCaFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-agent-ca/ca.pem"; + description = '' + Authority this daemon **issues** agent client certificates from, so + that an agent container can authenticate to the swarm secret store + under its own name. Read together with + {option}`services.hyperhive.deploy.swarm-controller.agentCaKeyFile`, + which is the private key it signs with. + + The mirror image of + {option}`services.hyperhive.deploy.swarm-controller.hiveClientCaFile`: + that one is an authority this daemon only *trusts by value*, so it is + public material and needs no key. This one signs, so it does. + + Leaving this `null` — the default — makes the module mint a + self-signed authority in `${agentCaDir}` on first boot and use that. + That is the ordinary shape: the store pins an authority per cert-auth + role, by value, so the authority agents are issued from does not have + to be the store's own PKI and does not have to live on the store's + host. Name a file here only when an operator issues agent leaves from + somewhere else; doing so turns the self-signing unit off. + ''; + }; + + agentCaKeyFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/swarm-agent-ca/ca-key.pem"; + description = '' + Private key for + {option}`services.hyperhive.deploy.swarm-controller.agentCaFile`. + Both or neither — an authority with no key signs nothing, and the + daemon refuses to start half-configured rather than looking ready. + + A path, never a value: the key's bytes in a nix expression land in + the world-readable nix store, permanently. + ''; + }; + queue = { clientSecretFile = lib.mkOption { type = lib.types.str; @@ -634,7 +709,10 @@ in services.hyperhive.swarm.otel.journaldUnits = [ "swarm-controller" "swarm-controller-credential" - ]; + ] + # Declared only where the unit exists — an entry for a unit that was + # never defined is a collector waiting on a journal that never speaks. + ++ lib.optional (haveAgentCa && selfSignAgentCa) "swarm-controller-agent-ca"; users.users.swarm-controller = { isSystemUser = true; @@ -721,6 +799,18 @@ in state directory. ''; } + { + assertion = + deployCfg.swarm-controller.agentCaFile == null || deployCfg.swarm-controller.agentCaKeyFile != null; + message = '' + services.hyperhive.deploy.swarm-controller.agentCaFile names an + authority but agentCaKeyFile is unset. + + The controller does not merely trust this authority, it issues + agent client certificates from it, so it needs the private key. + Set both, or set neither and let the module mint its own. + ''; + } ]; systemd.services.swarm-controller = { @@ -762,7 +852,16 @@ in ++ lib.optional ( haveBaoIdentity && deployCfg.bao.serverCaFile != null ) "bao-ca.pem:${deployCfg.bao.serverCaFile}" - ++ lib.optional haveHiveClientCa "hive-client-ca.pem:${deployCfg.swarm-controller.hiveClientCaFile}"; + ++ lib.optional haveHiveClientCa "hive-client-ca.pem:${deployCfg.swarm-controller.hiveClientCaFile}" + # The agent authority, key included — same shape and same reason as + # the store identity above: the key is root-owned `0600` and this + # daemon runs as `swarm-controller`. `swarm-controller-agent-ca` + # below is `requiredBy` this unit, so the files exist by the time + # systemd resolves these. + ++ lib.optionals haveAgentCa [ + "agent-ca.pem:${agentCaCert}" + "agent-ca-key.pem:${agentCaKey}" + ]; # The placeholder default that makes the above non-fatal. # `LoadCredential=` takes priority over `SetCredential=`, so this is @@ -935,5 +1034,50 @@ in ExecStart = "${pkgs.systemd}/bin/systemctl try-restart swarm-controller.service"; }; }; + + # The authority agent client leaves are issued from, minted here when the + # operator named none. Shape copied from ./glue-bao-tls.nix's + # `swarm-bao-pki`, including the rule that matters most: + # + # 🩸 Idempotent on ABSENCE, never on content. Re-issuing this CA would + # invalidate every agent leaf already published to the store AND every + # cert-auth role that pinned it by value, locking every agent container + # in the swarm out at once — on a rebuild that changed nothing an + # operator asked for. + # + # `before` + `requiredBy` rather than `after`: the daemon's + # `LoadCredential=` names these files by absolute path, and a + # `LoadCredential=` pointing at a file that is not there yet is fatal + # (`243/CREDENTIALS`), not a slow start. + systemd.services.swarm-controller-agent-ca = lib.mkIf (haveAgentCa && selfSignAgentCa) { + description = "mint the authority swarm agents' store certificates are issued from"; + before = [ "swarm-controller.service" ]; + requiredBy = [ "swarm-controller.service" ]; + path = [ + pkgs.openssl + pkgs.coreutils + ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + script = '' + set -euo pipefail + install -d -m 0700 ${agentCaDir} + + if [ ! -s ${agentCaCert} ]; then + # `pathlen:0` — this authority signs leaves and nothing else. An + # intermediate under it would be a second issuer for the one name + # space the store matches agents by. + openssl req -x509 -newkey rsa:4096 -nodes -sha256 -days 3650 \ + -keyout ${agentCaKey} -out ${agentCaCert} \ + -subj "/CN=swarm-agent-ca ${swarmDomain}" \ + -addext "basicConstraints=critical,CA:TRUE,pathlen:0" \ + -addext "keyUsage=critical,keyCertSign,cRLSign" + chmod 0600 ${agentCaKey} + chmod 0644 ${agentCaCert} + fi + ''; + }; }; } diff --git a/swarm-controller/Cargo.toml b/swarm-controller/Cargo.toml index 3bc1ea41..62d2777e 100644 --- a/swarm-controller/Cargo.toml +++ b/swarm-controller/Cargo.toml @@ -91,6 +91,15 @@ swarm-queue-client = { workspace = true, features = ["kv"] } # name and the object's shape are agreements between the two ends, and a # second spelling here would be a store this hive could not read. swarm-secret-client.workspace = true +# `agent_identity.rs` signs the per-agent client leaf the store authenticates +# an agent container by. The one runtime signer in this tree — every other CA +# here is a deploy-time `openssl` oneshot — because this one issues per agent +# creation and must keep the key it generates in memory, never on disk. +rcgen.workspace = true +# `rcgen::CertificateParams`'s validity window is a `time::OffsetDateTime`, +# which `agent_identity::validity` builds. Same workspace pin every other +# holder of a timestamp here uses. +time.workspace = true # `otel_http_client.rs`'s `AuthenticatedHttpClient` — an # `opentelemetry_http::HttpClient` impl authenticated with this crate's own # `swarm-queue-client` identity. Lives in this crate rather than diff --git a/swarm-controller/src/agent_identity.rs b/swarm-controller/src/agent_identity.rs new file mode 100644 index 00000000..8475222c --- /dev/null +++ b/swarm-controller/src/agent_identity.rs @@ -0,0 +1,420 @@ +//! One agent's own identity at the swarm's secret store: minted here, +//! published here, granted here — and, before the job node reports success, +//! **used** here. +//! +//! The swarm mints the agent's certificate so that no hive ever needs the +//! capability to mint one; the hive only carries it down. The controller is +//! the swarm-level service that does it because it already logs in to the +//! store, and its grant already covers exactly the three objects written here +//! (`swarm-bao.nix`'s `controllerPolicyText`: `create/update` on +//! `secret/data/swarm/agents/*`, on `sys/policies/acl/hive-*`, and on +//! `auth/cert/certs/hive-*`). No new authority is asked for anywhere. +//! +//! Four separate strings have to agree before an agent can authenticate: the +//! policy's name, the cert-auth role's name, the certificate's common name, +//! and the authority the role pins. That is the kind of agreement that holds +//! in review and fails in production — a mismatch is a 403 naming none of the +//! four — so [`mint_and_verify`] does not finish on a write. See +//! [`read_back_as_agent`]. +//! +//! ⚠️ The authority is **not** `/var/lib/swarm-bao-pki/ca-key.pem`. A +//! cert-auth role pins its authority by value, per role (see +//! [`SecretStore::write_cert_role`][swarm_secret_client::SecretStore::write_cert_role]), +//! so a role this daemon writes carries whatever authority this daemon hands +//! it — which is what lets the controller mint from its own CA on its own +//! host, with nothing co-located and no existing role changed. + +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result, bail}; +use swarm_secret_client::{ + SecretStore, + client::{DEFAULT_CERT_MOUNT, Settings}, + mtls, policy, +}; +use time::OffsetDateTime; + +/// File holding the authority agent leaves are issued from, as +/// `swarm-controller.nix` names it. Public material. +pub const ENV_AGENT_CA: &str = "SWARM_CONTROLLER_AGENT_CA_FILE"; + +/// File holding the private key for [`ENV_AGENT_CA`]. 🩸 A path, never a +/// value — the key's bytes must not reach a unit file or the nix store. +pub const ENV_AGENT_CA_KEY: &str = "SWARM_CONTROLLER_AGENT_CA_KEY_FILE"; + +/// How long a minted leaf is good for. +/// +/// Short enough that a leaked key is not permanent, long enough that the +/// absence of a renewal path is not immediately fatal. Nothing re-mints a leaf +/// today, so until a renewal path lands this is the interval after which an +/// operator re-runs agent creation. It is deliberately far shorter than the ten +/// years +/// `glue-bao-tls.nix` gives the store's own CA: that one is an authority +/// whose reissue invalidates every leaf under it, this one is a leaf. +/// Spelled in hours because `Duration::from_days` is not yet a stable `const +/// fn`; `read_policy`'s `RETRY_WINDOW` is the same workaround. +const LEAF_LIFETIME: Duration = Duration::from_hours(90 * 24); + +/// How far a leaf is backdated. +/// +/// The verifier is the store, on another machine: a certificate whose +/// `notBefore` is this exact instant is refused outright by a clock a second +/// behind ours, and the resulting error names a validity window rather than a +/// clock. +const CLOCK_SKEW: Duration = Duration::from_mins(5); + +/// The authority this daemon issues agent leaves from, loaded once at startup. +/// +/// ⚠️ **No `Debug` derive**, and the key field is private: this struct is +/// reachable from `WorkerDeps`, which is formatted nowhere today and is one +/// `#[derive(Debug)]` away from being formatted everywhere. +pub struct Authority { + /// The authority's certificate, PEM. Public material — it is also what + /// goes into each agent's cert-auth role and into each agent's published + /// credential. + ca_pem: String, + /// The authority's private key, PEM. Never logged, never published, + /// never leaves this struct. + key_pem: String, +} + +impl Authority { + /// Load the authority from the files [`ENV_AGENT_CA`] and + /// [`ENV_AGENT_CA_KEY`] name, or `None` when this host was given neither. + /// + /// `None` is a supported deployment, not a failure: it is the state every + /// controller is in before an operator has turned agent identities on, and + /// `run_swarm_node` reports it as that one node's named failure rather + /// than refusing to start the daemon. + /// + /// # Errors + /// When exactly one of the two variables is set — half an authority signs + /// nothing, and silently doing nothing about it is how a host ends up + /// looking configured — or when a named file cannot be read. + pub fn from_env() -> Result> { + match ( + std::env::var_os(ENV_AGENT_CA), + std::env::var_os(ENV_AGENT_CA_KEY), + ) { + (None, None) => Ok(None), + (Some(_), None) => bail!( + "{ENV_AGENT_CA} is set but {ENV_AGENT_CA_KEY} is not — an authority with no key signs nothing" + ), + (None, Some(_)) => bail!( + "{ENV_AGENT_CA_KEY} is set but {ENV_AGENT_CA} is not — a key with no certificate is not an authority" + ), + (Some(ca), Some(key)) => { + let ca_path = ca.to_string_lossy().into_owned(); + let key_path = key.to_string_lossy().into_owned(); + Ok(Some(Self { + ca_pem: std::fs::read_to_string(&ca_path).with_context(|| { + format!("reading the agent authority {ca_path} (from {ENV_AGENT_CA})") + })?, + key_pem: std::fs::read_to_string(&key_path).with_context(|| { + format!( + "reading the agent authority's key {key_path} (from {ENV_AGENT_CA_KEY})" + ) + })?, + })) + } + } + } + + /// Build one from PEM already in hand — the constructor a test uses, and + /// the one that keeps [`Authority::from_env`] the only place this process + /// reads a key off disk. + #[cfg(test)] + fn from_pem(ca_pem: String, key_pem: String) -> Self { + Self { ca_pem, key_pem } + } + + /// Issue a client leaf carrying `common_name`, returning `(certificate, + /// private key)` as PEM. + /// + /// `clientAuth` and nothing else: this certificate authenticates a + /// principal to the store and must not be usable to *serve* anything. + /// + /// # Errors + /// When the authority's own PEM will not parse, or the leaf will not sign. + fn mint_leaf(&self, common_name: &str) -> Result<(String, String)> { + let issuer_key = rcgen::KeyPair::from_pem(&self.key_pem) + .context("the agent authority's key is not a PEM key that can sign")?; + let issuer = rcgen::Issuer::from_ca_cert_pem(&self.ca_pem, issuer_key) + .context("the agent authority is not a PEM certificate that can issue")?; + + let (not_before, not_after) = validity(SystemTime::now(), LEAF_LIFETIME)?; + let mut params = rcgen::CertificateParams::default(); + params.distinguished_name = rcgen::DistinguishedName::new(); + params + .distinguished_name + .push(rcgen::DnType::CommonName, common_name); + params.is_ca = rcgen::IsCa::NoCa; + params.use_authority_key_identifier_extension = true; + params.key_usages = vec![ + rcgen::KeyUsagePurpose::DigitalSignature, + rcgen::KeyUsagePurpose::KeyEncipherment, + ]; + params.extended_key_usages = vec![rcgen::ExtendedKeyUsagePurpose::ClientAuth]; + params.not_before = not_before; + params.not_after = not_after; + + let leaf_key = rcgen::KeyPair::generate().context("generating the leaf's key")?; + let cert = params + .signed_by(&leaf_key, &issuer) + .with_context(|| format!("signing a leaf for {common_name}"))?; + Ok((cert.pem(), leaf_key.serialize_pem())) + } +} + +/// The validity window of a leaf minted at `now`, backdated by [`CLOCK_SKEW`]. +/// +/// A free function taking `now` rather than reading the clock itself, so the +/// arithmetic — the part that can be wrong by a factor of sixty — is testable +/// without waiting ninety days. +/// +/// # Errors +/// When the system clock is before the unix epoch, or so far past it that the +/// window will not fit a timestamp. +fn validity(now: SystemTime, lifetime: Duration) -> Result<(OffsetDateTime, OffsetDateTime)> { + let secs = now + .duration_since(UNIX_EPOCH) + .context("the system clock is before the unix epoch")? + .as_secs(); + let secs = i64::try_from(secs).context("the system clock is past what a timestamp holds")?; + let skew = i64::try_from(CLOCK_SKEW.as_secs()).expect("a five-minute constant fits an i64"); + let life = i64::try_from(lifetime.as_secs()).context("the leaf lifetime does not fit")?; + + let not_before = OffsetDateTime::from_unix_timestamp(secs - skew) + .context("the backdated start is not a representable time")?; + let not_after = OffsetDateTime::from_unix_timestamp(secs + life) + .context("the expiry is not a representable time")?; + Ok((not_before, not_after)) +} + +/// Give `agent` an identity at the store, and prove it works. +/// +/// Four store writes' worth of agreement, then the login that checks it: +/// +/// 1. mint a leaf whose common name is [`policy::agent_object_name`]; +/// 2. publish it at [`mtls::identity_path`], where the agent's hive collects +/// it under the hive's own certificate; +/// 3. write the ACL document [`policy::render_agent_with_queue`] renders — +/// read on this one agent's paths, plus the hive-shared queue credential +/// every agent container on `hive` already receives out of band; +/// 4. write the cert-auth role that ties the three together. +/// +/// Policy before role, for the reason `read_policy::provision` gives: the role +/// names the policy, so the other order leaves a window in which it points at +/// nothing. +/// +/// # Errors +/// Anything that stops one of those five steps, with the step named. A +/// failure here fails the job node and nothing else — the agent is still +/// created, exactly as capable as every agent is today. +pub async fn mint_and_verify(authority: &Authority, agent: &str, hive: &str) -> Result<()> { + let name = policy::agent_object_name(agent)?; + let path = mtls::identity_path(agent)?; + + let (cert, key) = authority.mint_leaf(&name)?; + let credential = mtls::Credential { + cert, + key, + ca: authority.ca_pem.clone(), + }; + + let store = crate::store::connect() + .await + .context("logging in to the swarm secret store")?; + store + .write(&path, &credential) + .await + .with_context(|| format!("publishing the agent identity at {path}"))?; + store + .write_policy(&name, &policy::render_agent_with_queue(agent, hive)?) + .await + .with_context(|| format!("writing the read policy {name}"))?; + store + .write_cert_role(DEFAULT_CERT_MOUNT, &name, &authority.ca_pem, &name, &name) + .await + .with_context(|| format!("writing the cert-auth role {name}"))?; + tracing::info!(agent, %path, role = %name, "agent store identity published"); + + read_back_as_agent(&credential, &name, &path).await?; + tracing::info!( + agent, + role = %name, + "agent store identity verified: the minted leaf logged in and read its own path" + ); + Ok(()) +} + +/// The consumer of everything [`mint_and_verify`] wrote: log in **as the +/// agent**, with the leaf just minted, and read back the path just published. +/// +/// The address and the store's CA come from this process's own `BAO_*` +/// environment; the *identity* deliberately does not — see +/// [`SecretStore::connect_with_identity`]. The freshly minted private key +/// never touches a filesystem. +/// +/// # Errors +/// When the store refuses the login (the role, the authority or the common +/// name disagree), when the read is denied (the policy does not cover the +/// path, or the role attached the wrong policy), or when what comes back is +/// not what went in. +async fn read_back_as_agent(credential: &mtls::Credential, role: &str, path: &str) -> Result<()> { + let settings = Settings::from_env() + .context("reading this daemon's own store settings for the read-back")?; + + // The concatenated blob `connect_with_identity` takes, built in memory. + let mut identity = credential.cert.clone().into_bytes(); + identity.push(b'\n'); + identity.extend_from_slice(credential.key.as_bytes()); + + let as_agent = + SecretStore::connect_with_identity(&settings, &identity, role, DEFAULT_CERT_MOUNT) + .await + .with_context(|| { + format!("logging in to the store as {role} with the leaf just minted") + })?; + let read_back: mtls::Credential = as_agent + .read(path) + .await + .with_context(|| format!("reading {path} back under {role}'s own token"))?; + + // Compared, not merely decoded: a successful read of an object written by + // some earlier run would otherwise pass this check while this run's leaf + // was the one nobody could use. Nothing about either value is printed. + if read_back != *credential { + bail!("the store returned a different object at {path} than the one just published"); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{Authority, CLOCK_SKEW, LEAF_LIFETIME, validity}; + use std::time::{Duration, SystemTime, UNIX_EPOCH}; + + /// A throwaway CA, minted in-process so no test needs a fixture file. + fn test_authority() -> Authority { + let key = rcgen::KeyPair::generate().expect("a key generates"); + let mut params = rcgen::CertificateParams::default(); + params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Constrained(0)); + params + .distinguished_name + .push(rcgen::DnType::CommonName, "swarm-agent-ca"); + let ca = params.self_signed(&key).expect("the CA self-signs"); + Authority::from_pem(ca.pem(), key.serialize_pem()) + } + + #[test] + fn the_window_is_backdated_by_the_skew_and_as_long_as_the_lifetime() { + // The arithmetic that is wrong by a factor of sixty if a unit slips. + let now = UNIX_EPOCH + Duration::from_secs(1_700_000_000); + let (before, after) = validity(now, LEAF_LIFETIME).expect("a plain instant is fine"); + assert_eq!(before.unix_timestamp(), 1_700_000_000 - 300); + assert_eq!(after.unix_timestamp(), 1_700_000_000 + 90 * 24 * 60 * 60); + assert_eq!(CLOCK_SKEW, Duration::from_mins(5)); + } + + #[test] + fn a_minted_leaf_starts_valid_and_expires() { + let now = i64::try_from( + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("the test host's clock is after 1970") + .as_secs(), + ) + .expect("and before the end of time"); + let (before, after) = validity(SystemTime::now(), LEAF_LIFETIME).expect("now is fine"); + assert!( + before.unix_timestamp() < now, + "a leaf usable only in the future is unusable" + ); + assert!( + after.unix_timestamp() > now, + "a leaf that has already expired authenticates nothing" + ); + // The rule `docs/swarm/credentials.md` states: no credential's + // renewal strategy may read NONE, which starts with it having an end. + // A decade-long leaf is that rule broken in a way review would miss. + assert!(after.unix_timestamp() - now < 3653 * 24 * 60 * 60); + } + + /// The four-strings agreement `mint_and_verify` rests on, checked on the + /// one of the four this process controls directly: the certificate really + /// does carry the common name the cert-auth role will be told to match. + #[test] + fn the_leaf_carries_the_agents_object_name_as_its_common_name() { + let name = swarm_secret_client::policy::agent_object_name("atlas").expect("legal"); + assert_eq!(name, "hive-agent-atlas"); + + let (cert, key) = test_authority().mint_leaf(&name).expect("the leaf signs"); + assert!(cert.contains("BEGIN CERTIFICATE"), "a PEM certificate"); + assert!(key.contains("PRIVATE KEY"), "a PEM key"); + + // Searched in the SIGNED DER rather than asserted on the params we + // built: the claim is that the name reached the bytes a verifier + // reads. A byte search rather than an X.509 parse because the whole + // crate would otherwise gain a parser dependency for one assertion — + // the name is a UTF8String in the subject DN, so it appears verbatim. + let body: String = cert + .lines() + .filter(|l| !l.starts_with("-----")) + .collect::>() + .join(""); + let der = base64::Engine::decode(&base64::engine::general_purpose::STANDARD, body) + .expect("the PEM body is base64"); + assert!( + der.windows(name.len()).any(|w| w == name.as_bytes()), + "the common name must be inside the signed certificate" + ); + + // And the pair is a usable client identity — the first thing + // `read_back_as_agent` does with it, in exactly this shape. + let mut identity = cert.into_bytes(); + identity.push(b'\n'); + identity.extend_from_slice(key.as_bytes()); + reqwest::Identity::from_pem(&identity).expect("the leaf and its key form a TLS identity"); + } + + /// A misconfiguration that would otherwise look like "not configured": + /// half an authority has to be an error, not a silent `None`. + /// + /// SAFETY: single-threaded mutation of two env vars no other test in this + /// crate reads, removed again before returning. + #[test] + fn half_an_authority_is_an_error_and_neither_half_is_absence() { + unsafe { + std::env::remove_var(super::ENV_AGENT_CA); + std::env::remove_var(super::ENV_AGENT_CA_KEY); + } + assert!( + Authority::from_env() + .expect("neither set is a supported shape") + .is_none(), + "a controller with no authority configured is not an error" + ); + + unsafe { std::env::set_var(super::ENV_AGENT_CA, "/nonexistent/ca.pem") } + // `.err().expect(..)` rather than `expect_err`: the `Ok` half is an + // `Authority`, which deliberately has no `Debug` (it holds a key). + let e = Authority::from_env() + .err() + .expect("a certificate with no key is half an authority"); + assert!(format!("{e:#}").contains(super::ENV_AGENT_CA_KEY), "{e:#}"); + + unsafe { + std::env::remove_var(super::ENV_AGENT_CA); + std::env::set_var(super::ENV_AGENT_CA_KEY, "/nonexistent/ca-key.pem"); + } + // `.err().expect(..)` rather than `expect_err`: the `Ok` half is an + // `Authority`, which deliberately has no `Debug` (it holds a key). + let e = Authority::from_env() + .err() + .expect("a key with no certificate is the other half"); + assert!(format!("{e:#}").contains(super::ENV_AGENT_CA), "{e:#}"); + + unsafe { std::env::remove_var(super::ENV_AGENT_CA_KEY) } + } +} diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index 99f43687..5594654e 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -40,6 +40,7 @@ use swarm_authelia_bridge_sock::BridgeResponse; use utoipa::{OpenApi, ToSchema}; use utoipa_axum::{router::OpenApiRouter, routes}; +mod agent_identity; mod agent_state_stream; mod agent_status; mod auth; @@ -94,6 +95,16 @@ enum SwarmNodeKind { /// swarm routes a deploy message to — it belongs on the node that /// sends that message, not on this one. InitAgentConfigRepo { agent: String }, + /// Mint `agent`'s own client certificate for the swarm secret store, + /// publish it there, grant it, and log in with it. See + /// `agent_identity::mint_and_verify` — including why this node does not + /// report success on a write. + /// + /// Carries the hive for a different reason than `TriggerDeploy` does: + /// not as an address, but because an agent's ACL document grants read on + /// its hive's shared queue credential, so the document cannot be rendered + /// without knowing which hive the agent belongs to. + MintAgentIdentity { hive: String, agent: String }, /// Tell `hive` to rebuild `agent`, by publishing on the swarm's deploy /// subject. The one node kind whose effect leaves this host. /// @@ -112,6 +123,7 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind { SwarmNodeKind::CreateForgeUser { .. } => "create_forge_user".to_owned(), SwarmNodeKind::AddRepoMember { .. } => "add_repo_member".to_owned(), SwarmNodeKind::InitAgentConfigRepo { .. } => "init_agent_config_repo".to_owned(), + SwarmNodeKind::MintAgentIdentity { .. } => "mint_agent_identity".to_owned(), SwarmNodeKind::TriggerDeploy { .. } => "trigger_deploy".to_owned(), } } @@ -131,7 +143,8 @@ impl hive_jobq_wire::WireNode for SwarmNodeKind { | SwarmNodeKind::InitAgentConfigRepo { agent } => { serde_json::json!({ "agent": agent }) } - SwarmNodeKind::TriggerDeploy { hive, agent } => { + SwarmNodeKind::TriggerDeploy { hive, agent } + | SwarmNodeKind::MintAgentIdentity { hive, agent } => { serde_json::json!({ "agent": agent, "hive": hive }) } } @@ -168,6 +181,11 @@ struct WorkerDeps { /// connection living there is an accident of construction order, not a /// claim that events are a kind of status. queue: Option, + /// The authority agent client leaves are issued from, loaded once at + /// startup because it holds a private key and a per-node re-read would be + /// a per-node chance to read one. `None` on a host the operator has not + /// given an authority — see `agent_identity::Authority::from_env`. + agent_ca: Option>, } /// Run a claimed node's actual work. Mirrors `hive-c0re/src/job_queue/ @@ -254,6 +272,20 @@ async fn run_swarm_node( Err(e) => Outcome::Failed(format!("{e:#}")), }, }, + SwarmNodeKind::MintAgentIdentity { hive, agent } => match deps.agent_ca { + None => Outcome::Failed( + "no agent certificate authority is configured on this host \ + (SWARM_CONTROLLER_AGENT_CA_FILE / SWARM_CONTROLLER_AGENT_CA_KEY_FILE unset), \ + so this agent has no identity at the swarm secret store" + .to_owned(), + ), + Some(authority) => { + match agent_identity::mint_and_verify(&authority, &agent, &hive).await { + Ok(()) => Outcome::Done, + Err(e) => Outcome::Failed(format!("{e:#}")), + } + } + }, SwarmNodeKind::TriggerDeploy { hive, agent } => match deps.queue { None => Outcome::Failed( "no swarm queue is configured on this host, so no hive can be told to deploy" @@ -1194,50 +1226,7 @@ async fn create_agent( .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let ids = sched - .insert_job(None, |b| { - let create_identity = b.node(SwarmNodeKind::CreateIdentity { - agent: agent.clone(), - }); - // A second, independent root: a forge user needs neither an - // authelia subject nor an existing repo, so it does not chain - // off `create_identity` (see the doc comment above). - let create_forge_user = b.node(SwarmNodeKind::CreateForgeUser { - agent: agent.clone(), - }); - let create_repo = b - .node(SwarmNodeKind::CreateRepo { - agent: agent.clone(), - }) - .after_ok(create_identity); - // `AddRepoMember` needs both parents: the repo to add a - // collaborator to, and the forge user to add as one — adding a - // nonexistent user is a Forgejo validation error, not - // an idempotent no-op. `InitAgentConfigRepo` needs only the - // repo — see the doc comment above for why. - let _add_repo_member = b - .node(SwarmNodeKind::AddRepoMember { - agent: agent.clone(), - }) - .after_ok(create_repo) - .after_ok(create_forge_user); - let init_config = b - .node(SwarmNodeKind::InitAgentConfigRepo { - agent: agent.clone(), - }) - .after_ok(create_repo); - // Last, and specifically after the config repo is seeded: the - // hive deploys by reading that repo, so a deploy asked for any - // earlier would find nothing to build. This is the edge that - // makes creating an agent at swarm level actually put it on a - // hive, rather than leaving a provisioned name nobody runs. - let _trigger_deploy = b - .node(SwarmNodeKind::TriggerDeploy { - hive: hive.clone(), - agent, - }) - .after_ok(init_config); - vec![create_identity.guid()] - }) + .insert_job(None, |b| declare_agent_job(b, &agent, &hive)) .map_err(|e| { error_problem( axum::http::StatusCode::INTERNAL_SERVER_ERROR, @@ -1253,6 +1242,101 @@ async fn create_agent( })) } +/// The authority agent leaves are issued from, or `None` on a host that was +/// given none. +/// +/// Same "log and carry on" shape as `main`'s other optional wiring: a +/// controller with no agent authority still serves everything else, and +/// `MintAgentIdentity` fails with a named reason rather than this process +/// refusing to start. The `Err` arm is worth its own warning — half an +/// authority, or a file that will not read, is a host that looks configured +/// and mints nothing. +fn load_agent_authority() -> Option> { + match agent_identity::Authority::from_env() { + Ok(authority) => authority.map(Arc::new), + Err(e) => { + tracing::warn!( + error = %format!("{e:#}"), + "agent certificate authority unusable; agents get no store identity here" + ); + None + } + } +} + +/// The sub-DAG one agent creation is: the nodes, and the edges between them. +/// +/// A function rather than a closure inside [`create_agent`] so the endpoint's +/// validation and the graph's shape can each be read without scrolling past +/// the other — and so the one handle the response reports is returned from +/// the place that decides which node it is. +fn declare_agent_job( + b: &hive_jobq::builder::JobBuilder, + agent: &str, + hive: &str, +) -> Vec { + let create_identity = b.node(SwarmNodeKind::CreateIdentity { + agent: agent.to_owned(), + }); + // A second, independent root: a forge user needs neither an authelia + // subject nor an existing repo, so it does not chain off + // `create_identity` (see the doc comment above). + let create_forge_user = b.node(SwarmNodeKind::CreateForgeUser { + agent: agent.to_owned(), + }); + let create_repo = b + .node(SwarmNodeKind::CreateRepo { + agent: agent.to_owned(), + }) + .after_ok(create_identity); + // `AddRepoMember` needs both parents: the repo to add a collaborator to, + // and the forge user to add as one — adding a nonexistent user is a + // Forgejo validation error, not an idempotent no-op. + // `InitAgentConfigRepo` needs only the repo — see the doc comment above + // for why. + let _add_repo_member = b + .node(SwarmNodeKind::AddRepoMember { + agent: agent.to_owned(), + }) + .after_ok(create_repo) + .after_ok(create_forge_user); + let init_config = b + .node(SwarmNodeKind::InitAgentConfigRepo { + agent: agent.to_owned(), + }) + .after_ok(create_repo); + // The agent's own identity at the swarm secret store, minted and + // published at swarm level so no hive ever needs the capability to mint + // one. After `create_identity` because the certificate names a principal + // the swarm has agreed exists — not because anything in the store reads + // authelia. + let mint_identity = b + .node(SwarmNodeKind::MintAgentIdentity { + hive: hive.to_owned(), + agent: agent.to_owned(), + }) + .after_ok(create_identity); + // Last, and specifically after the config repo is seeded: the hive + // deploys by reading that repo, so a deploy asked for any earlier would + // find nothing to build. This is the edge that makes creating an agent at + // swarm level actually put it on a hive, rather than leaving a + // provisioned name nobody runs. + // + // `after_any` on the mint, not `after_ok`: a hive cannot pass down a + // certificate the swarm has not published, so the deploy must not + // overtake the mint — but a host with no authority configured must still + // create agents exactly as it does today. `after_ok` there would turn an + // unconfigured option into an agent nobody runs. + let _trigger_deploy = b + .node(SwarmNodeKind::TriggerDeploy { + hive: hive.to_owned(), + agent: agent.to_owned(), + }) + .after_ok(init_config) + .after_any(mint_identity); + vec![create_identity.guid()] +} + /// Query params for `GET /api/jobq/graph` — `?states=` narrows to root /// groups in the named states, same shape `hive_jobq_wire::parse_states` /// parses. @@ -1579,6 +1663,7 @@ async fn main() -> Result<()> { auth: auth.clone(), forge: forge_client.clone(), queue: status.as_ref().map(|s| s.queue_client()), + agent_ca: load_agent_authority(), }; let jobq = Arc::new(Mutex::new(hive_jobq::scheduler::Scheduler::new( @@ -1945,6 +2030,7 @@ mod tests { auth: None, forge: None, queue: None, + agent_ca: None, }; let runner = hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| { @@ -1997,6 +2083,7 @@ mod tests { auth: None, forge: None, queue: None, + agent_ca: None, }; let runner = hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| { @@ -2018,6 +2105,130 @@ mod tests { ); } + /// Third sibling of the two above, and the same deliberate caveat: with + /// no authority configured this reaches only the + /// graceful-absence-is-failure branch. The happy path is a live store + /// and a real login, which is exactly why it is `mint_and_verify`'s own + /// job to prove it at agent-creation time rather than a unit test's. + /// + /// What this does pin is the degrade: a host that was never given an + /// authority fails this one node with a reason that names the two + /// variables, and creates the agent anyway. + #[tokio::test] + async fn mint_agent_identity_node_runs_end_to_end_and_fails_without_an_authority() { + let mut sched = hive_jobq::scheduler::Scheduler::new( + hive_jobq::Graph::new(), + hive_jobq::resources::ResourceTable::new(), + ); + let id = sched + .append( + SwarmNodeKind::MintAgentIdentity { + hive: "pr1ma".to_owned(), + agent: "atlas".to_owned(), + }, + Vec::new(), + None, + ) + .expect("insert"); + let sched = std::sync::Arc::new(std::sync::Mutex::new(sched)); + + let deps = WorkerDeps { + auth: None, + forge: None, + queue: None, + agent_ca: None, + }; + let runner = + hive_jobq::scheduler::Scheduler::claim_next(&sched, move |id, kind, builder| { + run_swarm_node(id, kind, builder, deps) + }) + .expect("the node just inserted is runnable"); + runner + .await + .1 + .expect("no growth declared, nothing to reject"); + + let guard = sched.lock().unwrap(); + let node = guard.graph().node(id).expect("node still present"); + assert_eq!(node.state, hive_jobq::State::Failed); + let error = node.error.as_deref().unwrap_or_default(); + assert!( + error.contains(crate::agent_identity::ENV_AGENT_CA) + && error.contains(crate::agent_identity::ENV_AGENT_CA_KEY), + "the reason must name both variables an operator has to set, got {error:?}" + ); + } + + /// The node carries the hive, and the viewer has to see it. The `data` + /// match is an or-pattern on purpose (see its own comment), and this is + /// the assertion that the new variant joined the two-field arm rather + /// than the agent-only one — a viewer silently missing the hive is the + /// failure that comment describes having already happened once. + #[test] + fn a_mint_node_renders_both_the_agent_and_the_hive() { + use hive_jobq_wire::WireNode as _; + + let kind = SwarmNodeKind::MintAgentIdentity { + hive: "pr1ma".to_owned(), + agent: "atlas".to_owned(), + }; + assert_eq!(kind.label(), "mint_agent_identity"); + let data = kind.data(1); + assert_eq!(data["agent"], "atlas"); + assert_eq!(data["hive"], "pr1ma"); + } + + /// The ordering the operator's ruling requires: a hive cannot pass down + /// a certificate the swarm has not published, so the deploy message must + /// not leave before the mint is terminal. + /// + /// Asserted on the graph `create_agent` builds, because the edge is one + /// line in a builder closure and its absence changes nothing observable + /// until a real agent boots without an identity. + #[tokio::test] + async fn the_deploy_waits_for_the_mint_and_is_not_cancelled_by_it() { + use hive_jobq_wire::WireNode as _; + + let (state, sched) = state_with_roster(); + let _queued = super::create_agent( + axum::extract::State(state), + axum::Json(super::CreateAgentRequest { + name: "atlas".to_owned(), + hive: "pr1ma".to_owned(), + }), + ) + .await + .expect("a hive in the roster must be accepted"); + + let guard = sched + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let graph = guard.graph(); + let id_of = |label: &str| { + graph + .nodes() + .find(|n| n.payload.label() == label) + .unwrap_or_else(|| panic!("the graph holds a {label} node")) + .id + }; + let mint = id_of("mint_agent_identity"); + let deploy = graph.node(id_of("trigger_deploy")).expect("just found"); + + let when = deploy + .deps + .iter() + .find_map(|d| match d { + hive_jobq::Dep::Node { id, when } if *id == mint => Some(*when), + _ => None, + }) + .expect("the deploy waits for the mint"); + assert!( + when.accepts(hive_jobq::TerminalState::Failed), + "an unconfigured authority must not cancel the deploy; this edge \ + has to be `after_any`, not `after_ok`" + ); + } + /// The socket must not share a directory with anything else, because /// the socket is `0666` and the directory is therefore the only access /// control it has. `/run/hyperhive` in particular holds hive-c0re's diff --git a/swarm-secret-client/src/client.rs b/swarm-secret-client/src/client.rs index 9f39fa9a..ad9deef3 100644 --- a/swarm-secret-client/src/client.rs +++ b/swarm-secret-client/src/client.rs @@ -112,7 +112,37 @@ impl SecretStore { let mut pem = read_file(ENV_CLIENT_CERT, &settings.cert_path)?; pem.push(b'\n'); pem.extend_from_slice(&read_file(ENV_CLIENT_KEY, &settings.key_path)?); - let identity = reqwest::Identity::from_pem(&pem).map_err(Error::Tls)?; + Self::connect_with_identity(settings, &pem, cert_role, cert_mount).await + } + + /// Log in at the store `settings` names, presenting an identity the caller + /// is already holding rather than one named by the environment. + /// + /// `identity_pem` is the certificate and its private key concatenated into + /// one PEM blob — the shape [`reqwest::Identity::from_pem`] takes, and the + /// shape [`SecretStore::connect`] builds out of the two files + /// [`ENV_CLIENT_CERT`] and [`ENV_CLIENT_KEY`] name. Only the address and + /// the CA are read out of `settings` here; its two identity paths are not + /// touched. + /// + /// This is what a **minter** needs. A process that has just issued a leaf + /// holds the bytes, and the safest place for a freshly minted private key + /// is the memory it was generated in: writing it to a file purely so that + /// a `Settings` could name it would put a key on disk for no other reason + /// than to log in with it once. + /// + /// # Errors + /// [`Error::Tls`] when `identity_pem` is not a usable certificate/key + /// pair, [`Error::Settings`] when the address will not parse, and + /// [`Error::Vault`] when the store refuses the login — which is what a + /// certificate no cert-auth role accepts looks like from here. + pub async fn connect_with_identity( + settings: &Settings, + identity_pem: &[u8], + cert_role: &str, + cert_mount: &str, + ) -> Result { + let identity = reqwest::Identity::from_pem(identity_pem).map_err(Error::Tls)?; let mut builder = VaultClientSettingsBuilder::default(); builder diff --git a/swarm-secret-client/src/lib.rs b/swarm-secret-client/src/lib.rs index 0987e34a..28fd566c 100644 --- a/swarm-secret-client/src/lib.rs +++ b/swarm-secret-client/src/lib.rs @@ -5,8 +5,8 @@ //! the rules every path obeys ([`path`]), the translation from this //! deployment's environment into a logged-in client ([`client`]), and, per kind //! of secret, the path it lives at together with the fields it holds -//! ([`matrix`], [`queue`]). Each of those is a thing the controller and a hive -//! must say identically, so it is said once here. +//! ([`matrix`], [`queue`], [`mtls`]). Each of those is a thing the controller +//! and a hive must say identically, so it is said once here. //! //! [`policy`] is the same kind of agreement seen from the other side: which of //! those paths a given principal's own token may read — a hive's, and an @@ -18,9 +18,13 @@ //! [`client`] is deliberately ignorant of all of it: it moves whatever type a //! caller names, so a second kind of secret is a new module beside [`matrix`] //! and not another field on a struct shared with it. +//! +//! [`mtls`] is the one module about reaching the store rather than about a +//! value inside it, and its doc explains why that is not circular. pub mod client; pub mod matrix; +pub mod mtls; pub mod path; pub mod policy; pub mod queue; diff --git a/swarm-secret-client/src/mtls.rs b/swarm-secret-client/src/mtls.rs new file mode 100644 index 00000000..a7be55fb --- /dev/null +++ b/swarm-secret-client/src/mtls.rs @@ -0,0 +1,173 @@ +//! The mTLS agreement: where an agent's own identity **at the store** lives, +//! and what the object at that path holds. +//! +//! The sibling of [`crate::matrix`] and [`crate::queue`], and the one that is +//! about reaching the store rather than about something kept inside it. An +//! agent's client certificate is minted at swarm level +//! (`swarm-controller::agent_identity`) and published here; the agent's hive +//! collects it under the **hive's** own certificate and hands it into the +//! container. +//! +//! 🔑 The recursion this shape looks like it has — *a credential fetched from +//! the store that is what opens the store* — is broken by who reads it. The +//! reader is the hive, never the agent, and a hive already holds its own leaf. +//! Nothing has to already be an agent in order to obtain an agent's identity. + +use serde::{Deserialize, Serialize}; + +use crate::{ + Error, + path::{Kind, principal_prefix}, +}; + +/// The path holding `agent`'s client certificate for the store itself. +/// +/// One path per agent with no further segment below it, unlike +/// [`crate::matrix::account_path`]: an agent has exactly one identity, and a +/// second one under a name would be a second principal wearing that name. +/// +/// # Errors +/// [`Error::PathSegment`] when `agent` contains anything but `[A-Za-z0-9_-]`, +/// which is what keeps one agent's name from addressing another's identity. +pub fn identity_path(agent: &str) -> Result { + let prefix = principal_prefix(Kind::Agent, agent)?; + Ok(format!("{prefix}/bao-mtls")) +} + +/// What the path holds: the leaf, its private key, and the authority the leaf +/// was issued from. +/// +/// All three, because a reader has to reconstruct a usable identity from the +/// store alone — the same requirement [`crate::queue::Credential`] states for +/// carrying its client id. The authority rides along so the cert-auth role +/// and the certificate cannot be delivered from two different sources and +/// silently disagree; it is public material, unlike the other two fields. +/// +/// ⚠️ **No `Debug` derive.** See the hand-written impl below: one of these +/// fields is a private key, and a derived `Debug` would put it in any log line +/// that ever formatted a node's payload. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Credential { + /// The agent's client certificate, PEM. The nix-side reader in an agent's + /// hive spells `bao kv get -field=cert`, so this name is load-bearing for + /// a reader this crate does not control. + pub cert: String, + + /// The private key for [`Credential::cert`], PEM. `bao kv get -field=key` + /// on the reading side, and `0600` the moment it lands on disk there. + pub key: String, + + /// The authority [`Credential::cert`] was issued from, PEM. Public + /// material: it is also what the store keeps inside the agent's cert-auth + /// role, by value (see [`crate::client::SecretStore::write_cert_role`]). + pub ca: String, +} + +impl std::fmt::Debug for Credential { + /// Redacts the key and summarises the two public fields by length. + /// + /// Hand-written rather than derived because the derive is the failure: + /// this type is carried through a job-graph node and an `anyhow` context + /// chain, both of which format whatever they are given. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Credential") + .field("cert", &format_args!("{} bytes of PEM", self.cert.len())) + .field("key", &"") + .field("ca", &format_args!("{} bytes of PEM", self.ca.len())) + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_agent_name_lands_under_its_own_principal_prefix() { + assert_eq!( + identity_path("atlas").expect("a plain name is legal"), + "swarm/agents/atlas/bao-mtls" + ); + } + + #[test] + fn a_traversal_in_the_agent_name_is_refused() { + let e = identity_path("../argus").expect_err("a traversal is not"); + assert!(matches!(e, Error::PathSegment { kind: "agent", .. }), "{e}"); + } + + #[test] + fn two_agents_never_share_a_path() { + assert_ne!( + identity_path("atlas").expect("legal"), + identity_path("argus").expect("legal") + ); + } + + /// The path sits under the same prefix an agent's policy grants + /// (`policy::render_agent`'s stanza is `swarm/agents//*`), which is + /// what makes an agent able to read its own identity back. + #[test] + fn an_agents_identity_is_inside_its_own_policy_stanza() { + let path = identity_path("atlas").expect("legal"); + let document = crate::policy::render_agent("atlas").expect("legal"); + let prefix = format!("{}/data/swarm/agents/atlas/", crate::path::MOUNT); + assert!( + path.starts_with("swarm/agents/atlas/"), + "the identity must sit under the agent's own prefix, got {path}" + ); + assert!( + document.contains(&format!("{prefix}*")), + "the agent's document must cover {prefix}*, got:\n{document}" + ); + } + + #[test] + fn the_object_round_trips_through_the_store_representation() { + let c = Credential { + cert: "-----BEGIN CERTIFICATE-----\n".to_owned(), + key: "-----BEGIN PRIVATE KEY-----\n".to_owned(), + ca: "-----BEGIN CERTIFICATE-----\n".to_owned(), + }; + let json = serde_json::to_string(&c).expect("serialises"); + assert_eq!( + serde_json::from_str::(&json).expect("deserialises"), + c + ); + } + + #[test] + fn the_field_names_the_nix_reader_asks_for_are_the_ones_written() { + let json = serde_json::to_value(Credential { + cert: "leaf".to_owned(), + key: "private".to_owned(), + ca: "authority".to_owned(), + }) + .expect("serialises"); + assert_eq!(json["cert"], "leaf"); + assert_eq!(json["key"], "private"); + assert_eq!(json["ca"], "authority"); + } + + /// The property the hand-written `Debug` exists for: a key that reaches a + /// log line is a key on a disk somebody else owns. + #[test] + fn formatting_the_credential_does_not_reveal_the_key() { + let rendered = format!( + "{:?}", + Credential { + cert: "leaf".to_owned(), + key: "SUPER-SECRET-KEY-MATERIAL".to_owned(), + ca: "authority".to_owned(), + } + ); + assert!( + !rendered.contains("SUPER-SECRET-KEY-MATERIAL"), + "the key must not survive formatting, got {rendered}" + ); + assert!( + rendered.contains(""), + "and the reader must be told it was withheld, got {rendered}" + ); + } +}