diff --git a/docs/networking/network.md b/docs/networking/network.md index cfb72b2e..0fb39114 100644 --- a/docs/networking/network.md +++ b/docs/networking/network.md @@ -181,9 +181,10 @@ namespace. By default agents can only reach the host on 80/443 (+53 DNS), so a host-side service on another port — for example a dev OTLP collector you want -agents to reach directly — is unreachable. (hyperhive's own telemetry -needs none of this: `otel.enable` opens its collector's port itself, and -`otel.endpoint` is the _upstream_, which no agent ever dials. See +agents to reach directly — is unreachable. (hyperhive's own services +need none of this: a module that means to be reachable from agents opens +its own port here, which `otel.enable` and `nats.enable` both do. Note +that `otel.endpoint` is the _upstream_, which no agent ever dials. See `docs/scheduler/observability.md`.) `services.hyperhive.network.exposeHostPorts = [ 4318 ];` opens each diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index a2309257..dc4dc517 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -143,6 +143,33 @@ let accessTokenSignedResponseAlg = "RS256"; }) hyperhiveCfg.swarm.hives; + # The identity an agent container presents to the swarm's queue. One per + # HIVE, not per agent, and that is the load-bearing choice rather than a + # shortcut: agents are created at **runtime**, so anything minted per agent + # would make creating one a config change plus an authelia reload. Keyed on + # the hive, this list's length tracks the roster above it — deploy-time, like + # every other entry here. + # + # ⚠️ The cost, stated rather than discovered later: every agent on a hive + # presents the SAME client id, so the broker can tell *hives* apart and not + # *agents*. Deliberate and deferred — closing it requires the client list to + # stop being static, which is the same problem the users-database writer + # already solves for identities. + # + # Mirrors `hiveClients` field for field so the two stay comparable. The + # signing algorithm is not load-bearing here — this client's only consumer is + # the queue's auth-callout responder, which *introspects* rather than + # verifying offline — but it matches its sibling rather than inventing a + # second answer to a question nobody asked. + agentClients = lib.mapAttrsToList (name: _: { + id = "${cfg.agentClientPrefix}${name}"; + description = "HyperHive agents on hive ${name}"; + kind = "machine"; + redirectUris = [ ]; + audience = [ "${cfg.agentClientPrefix}${name}" ]; + accessTokenSignedResponseAlg = "RS256"; + }) hyperhiveCfg.swarm.hives; + # `swarm-authelia-bridge`'s own identity — distinct from # `swarm-controller`'s (`swarm-controller.nix`'s `queueClientId`). A # resource server introspecting a token proves its OWN identity to the @@ -416,9 +443,21 @@ in default = deployCfg.nats.enable; defaultText = lib.literalExpression "services.hyperhive.deploy.nats.enable"; description = '' - Mint one machine client per hive in + Mint machine clients per hive in {option}`services.hyperhive.swarm.hives`, so each hive can - authenticate to swarm services as itself. + authenticate to swarm services as itself: `hive-` for the + hive's own daemons, and `agent-` for the agent containers + running on it. + + Two clients rather than one because they are not the same + principal — a hive's daemons run on the host and an agent runs + in a container the host hands a credential to, so a swarm + service has to be able to grant them different things. It is + one client per *hive* on the agent side, not per agent: agents + are created at runtime, and a per-agent client would make + creating one a config change plus an authelia reload. The cost + is that agents on a hive are indistinguishable from each other, + tracked as a follow-up rather than papered over. Defaults to whether the swarm message queue is enabled, because that is the first service that needs a hive to prove who it is. @@ -667,6 +706,27 @@ in ''; }; + agentClientPrefix = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = "agent-"; + description = '' + Prefix of the OAuth2 client id minted for the *agents* of each hive + in `services.hyperhive.swarm.hives` — agents on hive `alpha` all + present + `${config.services.hyperhive.swarm.authelia.agentClientPrefix}alpha`. + Read-only for the same reason as `hiveClientPrefix`, and read by the + same consumer with the same failure mode: a split spelling denies + every agent as a timeout. + + One id per hive rather than per agent, because agents are created at + runtime and a per-agent client would make creating one a config + change plus a reload. The consequence is that this identity says + *which hive* an agent belongs to and never *which agent* — the + broker cannot tell two agents on one hive apart. + ''; + }; + machine = lib.mkOption { type = lib.types.str; readOnly = true; @@ -828,7 +888,7 @@ in # wherever this module is, so `oidc.clients` is never actually empty # — see `oidcEnabled`'s comment above. services.hyperhive.swarm.authelia.oidc.clients = - lib.optionals cfg.oidc.hiveIdentities hiveClients + lib.optionals cfg.oidc.hiveIdentities (hiveClients ++ agentClients) ++ [ bridgeClient ]; # A redirect URI on a machine client is not harmless-but-unused: it diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index 84b9e930..90c7d0d0 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -233,6 +233,17 @@ in TCP port the queue listens on. 4222 is upstream's default and sits outside hyperhive's claimed ranges (dashboard 7000, forge 3000, matrix 8008, every agent in 8100..8999 via FNV-1a hash). + + Contributed to + `services.hyperhive.network.exposeHostPorts`, which opens it on + the bridge interface only — so it is reachable from agent + containers and not from the outside world. An agent connects to + `nats://:`; loopback inside a container is the + agent itself, not this host. + + Unlike {option}`monitorPort` and {option}`metricsPort`, the + address is not what bounds who may use this port: the queue's + `auth_callout` refuses every client it cannot identify. ''; }; @@ -564,6 +575,26 @@ in nats = "127.0.0.1:${toString cfg.metricsPort}"; }; + # Open the client port on the bridge, so agent containers can reach + # the queue. `privateNetwork = false` below means the server binds in + # the host netns, which is the precondition this option names — but + # sharing a netns is not reachability: the bridge interface is + # default-deny, so without this an agent's connect attempt is dropped + # by the firewall and looks exactly like every other NATS failure, + # a timeout. + # + # ⚠️ Bridge interface only — never the world. What makes it safe to + # open at all is that the queue is fail-closed: `auth_callout` admits + # nobody until the responder above answers for them, so an agent that + # reaches this port still has to present a token authelia vouches for. + # + # An agent connects to `nats://:`, NOT to + # `nats://127.0.0.1:` — inside a container loopback is the + # *agent*. Every url in this repo today is the loopback one and each + # is correct for its reader, because those readers share the host + # netns; an agent does not. + services.hyperhive.network.exposeHostPorts = [ cfg.port ]; + containers.swarm-nats = { autoStart = true; ephemeral = false; @@ -730,14 +761,15 @@ in # be the same string — which is why both come from one let. "--account ${lib.escapeShellArg clientAccount}" "--introspection-url ${lib.escapeShellArg introspectionUrl}" - # Both of these name a principal some OTHER module mints, - # so both are read out of that module rather than spelled + # Each of these names a principal some OTHER module mints, + # so each is read out of that module rather than spelled # again here — same argument as `--account` above, one # level wider. The responder denies a client id it does # not recognise, and a NATS denial arrives as a timeout, # so a drift here is silent at the point of change and # misattributed at the point of failure. "--hive-client-prefix ${lib.escapeShellArg autheliaCfg.hiveClientPrefix}" + "--agent-client-prefix ${lib.escapeShellArg autheliaCfg.agentClientPrefix}" "--reader-client ${lib.escapeShellArg controllerCfg.queueClientId}" ]; # Every credential arrives by `LoadCredential` and is named diff --git a/swarm-nats-auth/src/main.rs b/swarm-nats-auth/src/main.rs index a849419c..52969100 100644 --- a/swarm-nats-auth/src/main.rs +++ b/swarm-nats-auth/src/main.rs @@ -82,6 +82,16 @@ struct Args { #[arg(long, default_value = "hive-")] hive_client_prefix: String, + /// Client-id prefix that marks an agent container. `swarm-authelia.nix` + /// mints one machine client per roster entry as `agent-` — per + /// **hive**, not per agent, because agents are created at runtime and a + /// per-agent client would make creating one a config change plus a reload. + /// + /// So this identity says which hive an agent belongs to and never which + /// agent: two agents on one hive are indistinguishable to this responder. + #[arg(long, default_value = "agent-")] + agent_client_prefix: String, + /// Client ids allowed to read every hive's status. Repeatable. The /// default is the swarm controller, which is the only reader that exists. #[arg(long = "reader-client", default_values_t = [String::from("swarm-controller")])] @@ -96,6 +106,17 @@ struct Args { /// subject still lands inside that hive's own namespace. #[arg(long = "hive-publish-subject")] hive_publish_subjects: Vec, + + /// Subjects an agent may publish to, with `{hive}` standing for the hive + /// its identity names. Repeatable, empty by default. + /// + /// Empty means agents get **no grant at all** rather than a grant that can + /// do nothing — the identity exists, and what it may say is a deployment's + /// decision, not this responder's. The first consumer is the agent + /// terminal-event stream; until one is configured, an agent that connects + /// is refused, which is loud rather than silently over-broad. + #[arg(long = "agent-publish-subject")] + agent_publish_subjects: Vec, } /// Read a secret file and strip surrounding whitespace. @@ -140,9 +161,11 @@ async fn main() -> anyhow::Result<()> { // silently over-broad grant is not. let policy = policy::Policy::new( args.hive_client_prefix.clone(), + args.agent_client_prefix.clone(), swarm_queue_client::status::BUCKET.to_owned(), args.reader_clients.clone(), args.hive_publish_subjects.clone(), + args.agent_publish_subjects.clone(), )?; let http = reqwest::Client::new(); let issuer = nkeys::KeyPair::from_seed(&read_secret(&args.issuer_seed_file)?) diff --git a/swarm-nats-auth/src/policy.rs b/swarm-nats-auth/src/policy.rs index 6f11113f..0433b444 100644 --- a/swarm-nats-auth/src/policy.rs +++ b/swarm-nats-auth/src/policy.rs @@ -40,19 +40,24 @@ pub struct Permissions { #[derive(Debug, Clone)] pub struct Policy { hive_prefix: String, + agent_prefix: String, bucket: String, readers: Vec, extra_hive_subjects: Vec, + extra_agent_subjects: Vec, } -/// Placeholder replaced with the hive's own name in `extra_hive_subjects`. +/// Placeholder replaced with the hive's own name in `extra_hive_subjects` and +/// `extra_agent_subjects`. const HIVE_PLACEHOLDER: &str = "{hive}"; impl Policy { - /// `hive_prefix` is the client-id prefix that marks a hive, `bucket` the - /// KV bucket hives report status in, `readers` the client ids allowed to - /// read every hive's key, and `extra_hive_subjects` additional subjects a - /// hive may publish to (with `{hive}` standing for its own name). + /// `hive_prefix` is the client-id prefix that marks a hive and + /// `agent_prefix` the one that marks a hive's agent containers; `bucket` is + /// the KV bucket hives report status in, `readers` the client ids allowed + /// to read every hive's key, and `extra_hive_subjects` / + /// `extra_agent_subjects` additional subjects each may publish to (with + /// `{hive}` standing for the hive's own name). /// /// # Errors /// @@ -62,11 +67,18 @@ impl Policy { /// whose only purpose is to provide one. Returning a `Result` rather than /// checking at the call site is deliberate: it makes an unscoped policy /// unconstructible instead of merely unlikely. + /// + /// Two prefixes where one of them is a prefix of the other is refused for + /// the same reason: the arms are tried in order, so the overlap does not + /// error at match time — it silently hands one principal the other's + /// grant. pub fn new( hive_prefix: String, + agent_prefix: String, bucket: String, readers: Vec, extra_hive_subjects: Vec, + extra_agent_subjects: Vec, ) -> anyhow::Result { if let Some(bad) = extra_hive_subjects .iter() @@ -78,11 +90,40 @@ impl Policy { a per-hive namespace" ); } + // Same rule, same reason: an agent's identity is its *hive's*, so a + // template with no placeholder is one subject shared by every agent in + // the swarm rather than a per-hive one. + if let Some(bad) = extra_agent_subjects + .iter() + .find(|s| !s.contains(HIVE_PLACEHOLDER)) + { + anyhow::bail!( + "--agent-publish-subject {bad:?} contains no {HIVE_PLACEHOLDER}: every agent \ + in the swarm would be granted that exact subject, so it is a shared channel \ + rather than a per-hive namespace" + ); + } + // Checked here rather than trusted from nix, because the two prefixes + // are two flags and nothing downstream compares them. `permissions` + // tries the hive arm first, so an agent prefix of `""` — or either one + // being a prefix of the other — would route agents into the hive grant + // with no error anywhere: a client that authenticates fine and is + // granted more than it should have, which is the one failure this + // module must not have. + if hive_prefix.starts_with(&agent_prefix) || agent_prefix.starts_with(&hive_prefix) { + anyhow::bail!( + "--hive-client-prefix {hive_prefix:?} and --agent-client-prefix \ + {agent_prefix:?} overlap: one is a prefix of the other, so a client id \ + matching the longer one is granted by whichever rule is tried first" + ); + } Ok(Self { hive_prefix, + agent_prefix, bucket, readers, extra_hive_subjects, + extra_agent_subjects, }) } @@ -97,6 +138,15 @@ impl Policy { publish: self.hive_subjects(hive), }); } + // An agent's identity names its hive, never the agent — see + // `Self::agent_hive`. The grant is whatever the deployment configured + // for agents, expanded for that hive, and **nothing when it configured + // none**: an empty publish list would be a denial wearing a grant's + // shape, so this returns `None` and the client is refused outright. + if let Some(hive) = self.agent_hive(client_id) { + let publish = self.agent_subjects(hive); + return (!publish.is_empty()).then_some(Permissions { publish }); + } if self.readers.iter().any(|r| r == client_id) { return Some(Permissions { publish: self.reader_subjects(), @@ -134,6 +184,19 @@ impl Policy { .filter(|name| !name.is_empty()) } + /// The hive whose agents a client id names, when it names one. + /// + /// Deliberately the same shape as [`Self::hive_name`] one prefix over, and + /// every caveat above applies unchanged. One more on top: the agent client + /// is minted per **hive**, so this answers *whose agents*, never *which + /// agent*. Two agents on one hive are indistinguishable here — not a gap in + /// the parsing but the identity itself, which does not carry the agent. + fn agent_hive<'a>(&self, client_id: &'a str) -> Option<&'a str> { + client_id + .strip_prefix(&self.agent_prefix) + .filter(|name| !name.is_empty()) + } + fn stream(&self) -> String { format!("KV_{}", self.bucket) } @@ -314,6 +377,19 @@ impl Policy { subjects } + /// Subjects an agent of `hive` may publish to. + /// + /// No `JetStream` minimum, unlike [`Self::hive_subjects`]: a hive needs it + /// because it writes the status bucket, and an agent publishes to plain + /// subjects. Granting it anyway would be reasoning by analogy about a + /// permission set the module documents as minimal by removal. + fn agent_subjects(&self, hive: &str) -> Vec { + self.extra_agent_subjects + .iter() + .map(|s| s.replace(HIVE_PLACEHOLDER, hive)) + .collect() + } + fn reader_subjects(&self) -> Vec { let stream = self.stream(); let mut subjects = Self::jetstream_minimum().to_vec(); @@ -419,13 +495,30 @@ mod tests { fn policy() -> Policy { Policy::new( "hive-".to_owned(), + "agent-".to_owned(), "hive-status".to_owned(), vec!["swarm-controller".to_owned()], Vec::new(), + Vec::new(), ) .expect("the default policy is valid") } + /// The same policy with agents given something to publish, since the + /// default deployment configures no agent subject and an agent is then + /// refused outright. + fn policy_with_agent_subject() -> Policy { + Policy::new( + "hive-".to_owned(), + "agent-".to_owned(), + "hive-status".to_owned(), + vec!["swarm-controller".to_owned()], + Vec::new(), + vec!["$SWARM.term.{hive}.>".to_owned()], + ) + .expect("a per-hive agent template is valid") + } + #[test] fn the_reader_can_open_and_list_the_agent_status_bucket() { let subjects = policy().reader_subjects(); @@ -520,9 +613,11 @@ mod tests { // way to remove it - silently, and only in the deployment that set it. let err = Policy::new( "hive-".to_owned(), + "agent-".to_owned(), "hive-status".to_owned(), Vec::new(), vec!["$SWARM.events.all".to_owned()], + Vec::new(), ) .expect_err("a subject shared by every hive must not be accepted"); let msg = format!("{err}"); @@ -538,9 +633,11 @@ mod tests { // from the option being broken. Policy::new( "hive-".to_owned(), + "agent-".to_owned(), "hive-status".to_owned(), Vec::new(), vec!["$SWARM.events.{hive}.>".to_owned()], + Vec::new(), ) .expect("a per-hive template is the shape this option is for"); } @@ -944,13 +1041,88 @@ mod tests { // another's events even though its status key is scoped. let p = Policy::new( "hive-".to_owned(), + "agent-".to_owned(), "hive-status".to_owned(), Vec::new(), vec!["$SWARM.events.{hive}.>".to_owned()], + Vec::new(), ) .expect("a per-hive template is valid"); let g = p.permissions("hive-alpha").expect("a hive is admitted"); assert!(g.publish.contains(&"$SWARM.events.alpha.>".to_owned())); assert!(!g.publish.iter().any(|s| s.contains("{hive}"))); } + + #[test] + fn an_agent_with_no_configured_subject_is_refused() { + // The shipped default: the identity exists, what it may say does not. + // A grant with an empty publish list would let it connect and then + // fail on every publish, which reads as a broken queue rather than as + // a deployment that configured nothing. + assert!(policy().permissions("agent-alpha").is_none()); + } + + #[test] + fn an_agent_publishes_inside_its_own_hives_namespace() { + // The presence control for the test above - without it, "refused" + // would also be the answer if the agent arm never matched at all. + let g = policy_with_agent_subject() + .permissions("agent-alpha") + .expect("an agent with a configured subject is admitted"); + assert_eq!(g.publish, vec!["$SWARM.term.alpha.>".to_owned()]); + } + + #[test] + fn an_agent_does_not_get_its_hives_grant() { + // The whole point of a second prefix. An agent holds a credential that + // sits in a container; a hive's grant includes writing that hive's + // status key, which an agent must not be able to forge. + let g = policy_with_agent_subject() + .permissions("agent-alpha") + .expect("an agent is admitted"); + assert!(!g.publish.iter().any(|s| s.starts_with("$KV."))); + assert!(!g.publish.iter().any(|s| s.starts_with("$JS.API."))); + } + + #[test] + fn an_agent_subject_without_the_placeholder_is_refused() { + let err = Policy::new( + "hive-".to_owned(), + "agent-".to_owned(), + "hive-status".to_owned(), + Vec::new(), + Vec::new(), + vec!["$SWARM.term.all".to_owned()], + ) + .expect_err("a subject shared by every agent in the swarm is not per-hive"); + assert!( + format!("{err}").contains("$SWARM.term.all"), + "the error must name the offending template" + ); + } + + #[test] + fn overlapping_prefixes_are_refused() { + // 🩸 The arms are tried in order, so an overlap does not error at match + // time - it hands one principal the other's grant. Both directions, + // because which one wins depends only on the order above. + for (hive, agent) in [("hive-", "hive-agent-"), ("agent-x-", "agent-"), ("h", "h")] { + Policy::new( + hive.to_owned(), + agent.to_owned(), + "hive-status".to_owned(), + Vec::new(), + Vec::new(), + Vec::new(), + ) + .expect_err("prefixes where one contains the other must not construct"); + } + } + + #[test] + fn the_agent_prefix_alone_names_no_hive() { + // Same trap as `the_prefix_alone_names_no_hive`: `agent-` with nothing + // after it would expand `{hive}` to the empty string. + assert!(policy_with_agent_subject().permissions("agent-").is_none()); + } }