diff --git a/CLAUDE.md b/CLAUDE.md index 584e21b5..659d1e0d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -141,6 +141,13 @@ hand-maintained per-file tree drifts out of sync with the code. dependency-free wire types. - **`hive-metric/`** — small CLI to push a single labeled metric to the OTEL collector via the OpenTelemetry Rust SDK / OTLP HTTP exporter. +- **`swarm-controller/`** — swarm-level daemon, opt-in per host + (`services.hyperhive.swarm.controller.enable`). Where `hive-c0re` owns + the agents on **one** host, this owns what is true **across** hives; a + swarm runs one of them, so most hives leave it off. Serves HTTP over a + unix socket the gateway's nginx proxies to — ⚠️ **the socket's + directory is its access control**; the constraint that governs it is in + the crate's README, and a unit test pins the path. ### External dependencies with no directory here diff --git a/Cargo.lock b/Cargo.lock index 22171566..caba39bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4421,6 +4421,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "swarm-controller" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "syn" version = "1.0.109" diff --git a/Cargo.toml b/Cargo.toml index ea41f781..61341ce6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ members = [ "hive-sock-client", "hive-types", "hivectl", + "swarm-controller", ] [workspace.package] diff --git a/docs/swarm.md b/docs/swarm.md index addf88f5..c255d44a 100644 --- a/docs/swarm.md +++ b/docs/swarm.md @@ -235,6 +235,19 @@ one store, because the receiver keys destinations by *agent* so a migrating agent keeps one unbroken incremental chain. See [snapshot-store.md](snapshot-store.md). +## Swarm controller + +`services.hyperhive.swarm.controller.enable` runs the `swarm-controller` +daemon on this host. **Off by default and deliberately not derived from +`services.hyperhive.enable`**: a swarm has one controller, so enabling it +is a statement about swarm topology, not about whether hyperhive is +installed. Every hive runs `hive-c0re` (the agents on that host); one +hive additionally runs this (what is true across hives). + +What it serves, why it is a unix socket rather than a port, and the +socket-directory constraint that governs where `socketPath` may point: +[`swarm-controller/README.md`](../swarm-controller/README.md). + ## Cross-references - `docs/snapshot-store.md` — the swarm's `btrfs receive` endpoint, and diff --git a/flake.nix b/flake.nix index 87d4f7e4..62abd8ad 100644 --- a/flake.nix +++ b/flake.nix @@ -142,6 +142,9 @@ agentBaseToplevel = lib.mkDefault self.packages.x86_64-linux.agent-base-toplevel; managerToplevel = lib.mkDefault self.packages.x86_64-linux.ruth-toplevel; }; + services.hyperhive.swarm.controller.package = + lib.mkDefault + self.packages.${pkgs.stdenv.hostPlatform.system}.swarm-controller; services.hyperhive.gateway.swaggerUiTheme = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.swagger-ui-theme; diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index 2b87a421..3b5e41cc 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -21,6 +21,7 @@ ./hive-priv.nix ./hive-tls.nix ./otel.nix + ./swarm-controller.nix ./swarm-snapshot-store.nix ./swarm-wireguard.nix ./swarm.nix diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix new file mode 100644 index 00000000..9449e39b --- /dev/null +++ b/nix/host-modules/swarm-controller.nix @@ -0,0 +1,117 @@ +# The swarm-level controller daemon. Per-host opt-in: a swarm has one +# controller, so most hives leave this off and point at the hive that +# runs it. Distinct from hive-c0re, which every hive runs — c0re owns +# the agents on one host, this owns what is true across hives. +# +# Serves HTTP over a unix socket rather than a TCP port: the gateway's +# nginx is the only intended client, it reaches the socket through a +# bind-mount, and a socket that is never bound to an address cannot be +# reached from off-host by mistake. +{ + lib, + config, + ... +}: +let + cfg = config.services.hyperhive.swarm.controller; +in +{ + options.services.hyperhive.swarm.controller = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Run the swarm-controller daemon on this host. Off by default and + deliberately not derived from `services.hyperhive.enable`: a swarm + has one controller, so enabling it per hive is a decision about + swarm topology, not about whether hyperhive is installed. + ''; + }; + + package = lib.mkOption { + type = lib.types.package; + defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-controller"; + description = '' + swarm-controller package. Wired by default from this flake's own + package set (see `flake.nix`); override to run a different build. + ''; + }; + + socketPath = lib.mkOption { + type = lib.types.str; + default = "/run/swarm-controller/controller.sock"; + description = '' + Unix socket the daemon serves on, and the path the gateway's nginx + proxies to. + + The **directory** is the access control here, not the socket mode: + the socket itself is `0666` (nginx runs as another user, and + `connect(2)` needs write), exactly as hive-c0re publishes the + per-agent sockets. What keeps that safe is that the directory holds + one socket and is bind-mounted into one container. Moving this path + under a directory that carries anything else — `/run/hyperhive`, + which holds the host admin socket, above all — hands whatever else + lives there to every consumer that mounts it. + + Changing this therefore means re-checking the gateway bind-mount, + not just the daemon. + ''; + }; + }; + + config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) { + users.users.swarm-controller = { + isSystemUser = true; + group = "swarm-controller"; + description = "hyperhive swarm-controller daemon"; + }; + users.groups.swarm-controller = { }; + + systemd.services.swarm-controller = { + description = "hyperhive swarm-level controller daemon"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + + serviceConfig = { + ExecStart = "${cfg.package}/bin/swarm-controller"; + User = "swarm-controller"; + Group = "swarm-controller"; + Restart = "on-failure"; + RestartSec = "5s"; + + # `/run/swarm-controller` — its own directory, holding only the + # socket. See `socketPath`'s description for why that is a security + # property and not tidiness. + RuntimeDirectory = "swarm-controller"; + # 0751: traverse-only for others, so the gateway's nginx can reach + # the socket path without being able to list the directory. Same + # shape (and same reason) as hive-c0re's runtime dir. + RuntimeDirectoryMode = "0751"; + # Preserved across restarts so the bind-mount source never vanishes + # from under a running gateway container. The daemon unlinks a stale + # socket on start, which is what makes preservation safe. + RuntimeDirectoryPreserve = "yes"; + + StateDirectory = "swarm-controller"; + StateDirectoryMode = "0750"; + + # Nothing here needs a writable filesystem, real privileges, or a + # view of the rest of the machine; the daemon reads its socket path + # from config and serves. + PrivateTmp = true; + ProtectSystem = "strict"; + ProtectHome = true; + NoNewPrivileges = true; + PrivateDevices = true; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectControlGroups = true; + RestrictAddressFamilies = [ + "AF_UNIX" + ]; + }; + + environment.SWARM_CONTROLLER_SOCKET = cfg.socketPath; + }; + }; +} diff --git a/nix/packages/default.nix b/nix/packages/default.nix index 70c5baa7..6713ae6e 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -133,6 +133,16 @@ in } // binPkgs // { + # Swarm-level controller daemon. Deliberately NOT in `daemonBins`: + # that list is the core stack — the binaries hive-c0re and the agent + # harness are made of — and it drives the `default` bundle that + # `services.hyperhive.c0re.package` points at. This daemon is a + # separate swarm-scoped service with its own module and its own + # `package` option, and one hive in a swarm runs it, so folding it + # into the core bundle would put it in every hive's closure to no end. + # Uses the same per-bin extractor, just bound on its own. + swarm-controller = mkBinPackage "swarm-controller" "hyperhive swarm-level controller daemon"; + # Bundled browser assets — see ./frontend.nix. Output is # $out/{dashboard,agent}/ which the Rust binaries serve via # tower_http::ServeDir. diff --git a/swarm-controller/Cargo.toml b/swarm-controller/Cargo.toml new file mode 100644 index 00000000..877efecd --- /dev/null +++ b/swarm-controller/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "swarm-controller" +version.workspace = true +readme = "README.md" +edition.workspace = true + +[[bin]] +name = "swarm-controller" +path = "src/main.rs" + +[dependencies] +anyhow.workspace = true +axum.workspace = true +tokio.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true + +[lints] +workspace = true diff --git a/swarm-controller/README.md b/swarm-controller/README.md new file mode 100644 index 00000000..0d896e85 --- /dev/null +++ b/swarm-controller/README.md @@ -0,0 +1,60 @@ +# swarm-controller + +The **swarm-level** daemon. Where `hive-c0re` owns the agents on one host, this +owns what is true *across* hives — so a swarm runs one of them and most hives +leave it off. + +Opt-in per host via `services.hyperhive.swarm.controller.enable`, which is +deliberately **not** derived from `services.hyperhive.enable`: turning it on is +a statement about swarm topology, not about whether hyperhive is installed. + +## What it does today + +Serves one `/health` endpoint and holds no state. + +That is the whole intent of the first slice. The point is to make the *unit* +real — service user, runtime and state directories, socket, nginx +reachability — so the swarm-level surfaces that follow have somewhere to land. +Inventing those surfaces before they are agreed would bake in a shape nobody +chose. See #3066 and the `hyperhive.swarm` consolidation epic. + +## Why a unix socket, not a port + +The hive-gateway's nginx is the only intended client and reaches the socket +through a bind-mount. A listener that is never bound to an address cannot be +reached from off-host by mistake. + +The socket path is `services.hyperhive.swarm.controller.socketPath`, default +`/run/swarm-controller/controller.sock`, exported to the process as +`SWARM_CONTROLLER_SOCKET`. + +## ⚠️ The socket's directory is its access control + +The socket is `0666`. It has to be: nginx runs as a different user and +`connect(2)` needs write. This matches how `hive-c0re` publishes the per-agent +sockets, and rests on the same argument — *"the bind source dir is per-agent on +host so blast radius is unchanged."* + +What keeps that safe is that the directory holds **one** socket and is +bind-mounted into **one** container. So: + +> **Never point `socketPath` at a directory that carries anything else.** +> `/run/hyperhive` above all — it holds `host.sock`, the host **admin** socket. +> Mounting that directory to reach this socket would hand the gateway container +> the admin socket along with it. + +Changing `socketPath` therefore means re-checking the gateway bind-mount, not +just the daemon. A unit test pins the default path so a tidying edit fails +instead of reviewing cleanly. + +`RuntimeDirectoryPreserve=yes` and the daemon's stale-socket unlink on start are +a **pair**: preserving the directory without the unlink means `bind` fails with +`EADDRINUSE` after a restart. + +## Packaging + +Built by the workspace derivation and extracted as its own package +(`nix build .#swarm-controller`). Deliberately **not** in `nix/packages`' +`daemonBins` — that list is the core stack and drives the bundle +`services.hyperhive.c0re.package` points at, so folding this in would put a +swarm-scoped service into every hive's closure. diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs new file mode 100644 index 00000000..aefb3a75 --- /dev/null +++ b/swarm-controller/src/main.rs @@ -0,0 +1,117 @@ +//! Swarm-level controller daemon. Runs as the unprivileged +//! `swarm-controller` user on whichever host the operator flips +//! `services.hyperhive.swarm.controller.enable` on, and serves HTTP over a +//! unix socket that the hive-gateway's nginx proxies to. +//! +//! **Today it serves one endpoint and owns no state.** That is deliberate: +//! this slice exists to make the *unit* real — service user, runtime and +//! state directories, socket, nginx reachability — so the swarm-level +//! surfaces that follow have somewhere to land. Guessing those surfaces +//! now would bake in a shape nobody has agreed to. +//! +//! Distinct from `hive-c0re`, which is per-hive: c0re owns the agents on +//! one host, this owns what is true across hives. + +use std::os::unix::fs::PermissionsExt as _; +use std::path::PathBuf; + +use anyhow::{Context, Result}; +use axum::{Router, routing::get}; + +/// Where the daemon binds, overridable via `SWARM_CONTROLLER_SOCKET`. +/// +/// A compiled-in default is legitimate here and is *not* the mistake that +/// a hardcoded remote address would be: this is a path this process +/// **creates**, not an address it hopes to find something at. systemd's +/// `RuntimeDirectory=swarm-controller` makes the parent exist before +/// `ExecStart`, so the default names a directory the unit just produced. +/// +/// The directory is its own — deliberately not shared with hive-c0re's +/// `/run/hyperhive`. nginx reaches a unix upstream by having the socket's +/// *directory* bind-mounted into the gateway container, so co-locating +/// this socket with c0re's admin socket would hand the gateway the admin +/// socket along with it. +const DEFAULT_SOCKET: &str = "/run/swarm-controller/controller.sock"; + +fn socket_path() -> PathBuf { + std::env::var_os("SWARM_CONTROLLER_SOCKET") + .map_or_else(|| PathBuf::from(DEFAULT_SOCKET), PathBuf::from) +} + +/// Liveness probe. Returns the build's version so an operator can tell +/// *which* controller answered without shelling onto the host. +async fn health() -> &'static str { + concat!("swarm-controller ", env!("CARGO_PKG_VERSION"), "\n") +} + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .init(); + + let path = socket_path(); + + // `RuntimeDirectoryPreserve=yes` keeps the directory across a restart, + // so a socket file from the previous run can outlive the process that + // owned it and `bind` would fail with EADDRINUSE. Unlinking a stale + // socket is safe precisely because the directory is ours alone: nothing + // else can have put a file at this path. + if let Err(e) = std::fs::remove_file(&path) + && e.kind() != std::io::ErrorKind::NotFound + { + return Err(e).with_context(|| format!("clearing stale socket at {}", path.display())); + } + + let listener = tokio::net::UnixListener::bind(&path) + .with_context(|| format!("binding {}", path.display()))?; + + // `bind` leaves the socket 0755, and connecting needs write — the + // gateway's nginx is a different user, so it would be locked out. + // 0666 matches how hive-c0re publishes the per-agent sockets + // (`socket_server::start`), and rests on the same argument: **the + // containing directory is the access control, not the socket mode.** + // This directory holds one socket and is bind-mounted into exactly + // one container. That is also why it must not be shared with + // hive-c0re's `/run/hyperhive` — with a 0666 socket, a directory + // that carries more than it should is the whole vulnerability. + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o666)) + .with_context(|| format!("chmod {}", path.display()))?; + tracing::info!(socket = %path.display(), "swarm-controller listening"); + + let app = Router::new().route("/health", get(health)); + axum::serve(listener, app) + .await + .context("serving swarm-controller") +} + +#[cfg(test)] +mod tests { + use super::DEFAULT_SOCKET; + use std::path::Path; + + /// The socket must not share a directory with anything else, because + /// the socket is `0666` and the directory is therefore the only access + /// control it has. `/run/hyperhive` in particular holds hive-c0re's + /// **admin** socket, and nginx reaches a unix upstream by mounting the + /// socket's whole directory into the gateway container. + /// + /// A test rather than a comment: the failure this guards against is a + /// one-word edit that looks tidier and reads fine in review. + #[test] + fn socket_lives_in_its_own_runtime_dir() { + let parent = Path::new(DEFAULT_SOCKET) + .parent() + .expect("socket path has a parent directory"); + assert_eq!( + parent, + Path::new("/run/swarm-controller"), + "the socket's directory is its access control — moving it under a shared \ + directory (notably /run/hyperhive, which holds the host admin socket) \ + exposes everything else in that directory to the gateway container" + ); + } +}