diff --git a/CLAUDE.md b/CLAUDE.md index 659d1e0d..584e21b5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -141,13 +141,6 @@ hand-maintained per-file tree drifts out of sync with the code. dependency-free wire types. - **`hive-metric/`** — small CLI to push a single labeled metric to the OTEL collector via the OpenTelemetry Rust SDK / OTLP HTTP exporter. -- **`swarm-controller/`** — swarm-level daemon, opt-in per host - (`services.hyperhive.swarm.controller.enable`). Where `hive-c0re` owns - the agents on **one** host, this owns what is true **across** hives; a - swarm runs one of them, so most hives leave it off. Serves HTTP over a - unix socket the gateway's nginx proxies to — ⚠️ **the socket's - directory is its access control**; the constraint that governs it is in - the crate's README, and a unit test pins the path. ### External dependencies with no directory here diff --git a/Cargo.lock b/Cargo.lock index caba39bb..22171566 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4421,17 +4421,6 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" -[[package]] -name = "swarm-controller" -version = "0.1.0" -dependencies = [ - "anyhow", - "axum", - "tokio", - "tracing", - "tracing-subscriber", -] - [[package]] name = "syn" version = "1.0.109" diff --git a/Cargo.toml b/Cargo.toml index 61341ce6..ea41f781 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,6 @@ members = [ "hive-sock-client", "hive-types", "hivectl", - "swarm-controller", ] [workspace.package] diff --git a/docs/swarm.md b/docs/swarm.md index c255d44a..addf88f5 100644 --- a/docs/swarm.md +++ b/docs/swarm.md @@ -235,19 +235,6 @@ one store, because the receiver keys destinations by *agent* so a migrating agent keeps one unbroken incremental chain. See [snapshot-store.md](snapshot-store.md). -## Swarm controller - -`services.hyperhive.swarm.controller.enable` runs the `swarm-controller` -daemon on this host. **Off by default and deliberately not derived from -`services.hyperhive.enable`**: a swarm has one controller, so enabling it -is a statement about swarm topology, not about whether hyperhive is -installed. Every hive runs `hive-c0re` (the agents on that host); one -hive additionally runs this (what is true across hives). - -What it serves, why it is a unix socket rather than a port, and the -socket-directory constraint that governs where `socketPath` may point: -[`swarm-controller/README.md`](../swarm-controller/README.md). - ## Cross-references - `docs/snapshot-store.md` — the swarm's `btrfs receive` endpoint, and diff --git a/flake.nix b/flake.nix index 62abd8ad..87d4f7e4 100644 --- a/flake.nix +++ b/flake.nix @@ -142,9 +142,6 @@ agentBaseToplevel = lib.mkDefault self.packages.x86_64-linux.agent-base-toplevel; managerToplevel = lib.mkDefault self.packages.x86_64-linux.ruth-toplevel; }; - services.hyperhive.swarm.controller.package = - lib.mkDefault - self.packages.${pkgs.stdenv.hostPlatform.system}.swarm-controller; services.hyperhive.gateway.swaggerUiTheme = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.swagger-ui-theme; diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index 3b5e41cc..2b87a421 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -21,7 +21,6 @@ ./hive-priv.nix ./hive-tls.nix ./otel.nix - ./swarm-controller.nix ./swarm-snapshot-store.nix ./swarm-wireguard.nix ./swarm.nix diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix deleted file mode 100644 index 9449e39b..00000000 --- a/nix/host-modules/swarm-controller.nix +++ /dev/null @@ -1,117 +0,0 @@ -# The swarm-level controller daemon. Per-host opt-in: a swarm has one -# controller, so most hives leave this off and point at the hive that -# runs it. Distinct from hive-c0re, which every hive runs — c0re owns -# the agents on one host, this owns what is true across hives. -# -# Serves HTTP over a unix socket rather than a TCP port: the gateway's -# nginx is the only intended client, it reaches the socket through a -# bind-mount, and a socket that is never bound to an address cannot be -# reached from off-host by mistake. -{ - lib, - config, - ... -}: -let - cfg = config.services.hyperhive.swarm.controller; -in -{ - options.services.hyperhive.swarm.controller = { - enable = lib.mkOption { - type = lib.types.bool; - default = false; - description = '' - Run the swarm-controller daemon on this host. Off by default and - deliberately not derived from `services.hyperhive.enable`: a swarm - has one controller, so enabling it per hive is a decision about - swarm topology, not about whether hyperhive is installed. - ''; - }; - - package = lib.mkOption { - type = lib.types.package; - defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-controller"; - description = '' - swarm-controller package. Wired by default from this flake's own - package set (see `flake.nix`); override to run a different build. - ''; - }; - - socketPath = lib.mkOption { - type = lib.types.str; - default = "/run/swarm-controller/controller.sock"; - description = '' - Unix socket the daemon serves on, and the path the gateway's nginx - proxies to. - - The **directory** is the access control here, not the socket mode: - the socket itself is `0666` (nginx runs as another user, and - `connect(2)` needs write), exactly as hive-c0re publishes the - per-agent sockets. What keeps that safe is that the directory holds - one socket and is bind-mounted into one container. Moving this path - under a directory that carries anything else — `/run/hyperhive`, - which holds the host admin socket, above all — hands whatever else - lives there to every consumer that mounts it. - - Changing this therefore means re-checking the gateway bind-mount, - not just the daemon. - ''; - }; - }; - - config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) { - users.users.swarm-controller = { - isSystemUser = true; - group = "swarm-controller"; - description = "hyperhive swarm-controller daemon"; - }; - users.groups.swarm-controller = { }; - - systemd.services.swarm-controller = { - description = "hyperhive swarm-level controller daemon"; - wantedBy = [ "multi-user.target" ]; - after = [ "network.target" ]; - - serviceConfig = { - ExecStart = "${cfg.package}/bin/swarm-controller"; - User = "swarm-controller"; - Group = "swarm-controller"; - Restart = "on-failure"; - RestartSec = "5s"; - - # `/run/swarm-controller` — its own directory, holding only the - # socket. See `socketPath`'s description for why that is a security - # property and not tidiness. - RuntimeDirectory = "swarm-controller"; - # 0751: traverse-only for others, so the gateway's nginx can reach - # the socket path without being able to list the directory. Same - # shape (and same reason) as hive-c0re's runtime dir. - RuntimeDirectoryMode = "0751"; - # Preserved across restarts so the bind-mount source never vanishes - # from under a running gateway container. The daemon unlinks a stale - # socket on start, which is what makes preservation safe. - RuntimeDirectoryPreserve = "yes"; - - StateDirectory = "swarm-controller"; - StateDirectoryMode = "0750"; - - # Nothing here needs a writable filesystem, real privileges, or a - # view of the rest of the machine; the daemon reads its socket path - # from config and serves. - PrivateTmp = true; - ProtectSystem = "strict"; - ProtectHome = true; - NoNewPrivileges = true; - PrivateDevices = true; - ProtectKernelTunables = true; - ProtectKernelModules = true; - ProtectControlGroups = true; - RestrictAddressFamilies = [ - "AF_UNIX" - ]; - }; - - environment.SWARM_CONTROLLER_SOCKET = cfg.socketPath; - }; - }; -} diff --git a/nix/packages/default.nix b/nix/packages/default.nix index 6713ae6e..70c5baa7 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -133,16 +133,6 @@ in } // binPkgs // { - # Swarm-level controller daemon. Deliberately NOT in `daemonBins`: - # that list is the core stack — the binaries hive-c0re and the agent - # harness are made of — and it drives the `default` bundle that - # `services.hyperhive.c0re.package` points at. This daemon is a - # separate swarm-scoped service with its own module and its own - # `package` option, and one hive in a swarm runs it, so folding it - # into the core bundle would put it in every hive's closure to no end. - # Uses the same per-bin extractor, just bound on its own. - swarm-controller = mkBinPackage "swarm-controller" "hyperhive swarm-level controller daemon"; - # Bundled browser assets — see ./frontend.nix. Output is # $out/{dashboard,agent}/ which the Rust binaries serve via # tower_http::ServeDir. diff --git a/swarm-controller/Cargo.toml b/swarm-controller/Cargo.toml deleted file mode 100644 index 877efecd..00000000 --- a/swarm-controller/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "swarm-controller" -version.workspace = true -readme = "README.md" -edition.workspace = true - -[[bin]] -name = "swarm-controller" -path = "src/main.rs" - -[dependencies] -anyhow.workspace = true -axum.workspace = true -tokio.workspace = true -tracing.workspace = true -tracing-subscriber.workspace = true - -[lints] -workspace = true diff --git a/swarm-controller/README.md b/swarm-controller/README.md deleted file mode 100644 index 0d896e85..00000000 --- a/swarm-controller/README.md +++ /dev/null @@ -1,60 +0,0 @@ -# swarm-controller - -The **swarm-level** daemon. Where `hive-c0re` owns the agents on one host, this -owns what is true *across* hives — so a swarm runs one of them and most hives -leave it off. - -Opt-in per host via `services.hyperhive.swarm.controller.enable`, which is -deliberately **not** derived from `services.hyperhive.enable`: turning it on is -a statement about swarm topology, not about whether hyperhive is installed. - -## What it does today - -Serves one `/health` endpoint and holds no state. - -That is the whole intent of the first slice. The point is to make the *unit* -real — service user, runtime and state directories, socket, nginx -reachability — so the swarm-level surfaces that follow have somewhere to land. -Inventing those surfaces before they are agreed would bake in a shape nobody -chose. See #3066 and the `hyperhive.swarm` consolidation epic. - -## Why a unix socket, not a port - -The hive-gateway's nginx is the only intended client and reaches the socket -through a bind-mount. A listener that is never bound to an address cannot be -reached from off-host by mistake. - -The socket path is `services.hyperhive.swarm.controller.socketPath`, default -`/run/swarm-controller/controller.sock`, exported to the process as -`SWARM_CONTROLLER_SOCKET`. - -## ⚠️ The socket's directory is its access control - -The socket is `0666`. It has to be: nginx runs as a different user and -`connect(2)` needs write. This matches how `hive-c0re` publishes the per-agent -sockets, and rests on the same argument — *"the bind source dir is per-agent on -host so blast radius is unchanged."* - -What keeps that safe is that the directory holds **one** socket and is -bind-mounted into **one** container. So: - -> **Never point `socketPath` at a directory that carries anything else.** -> `/run/hyperhive` above all — it holds `host.sock`, the host **admin** socket. -> Mounting that directory to reach this socket would hand the gateway container -> the admin socket along with it. - -Changing `socketPath` therefore means re-checking the gateway bind-mount, not -just the daemon. A unit test pins the default path so a tidying edit fails -instead of reviewing cleanly. - -`RuntimeDirectoryPreserve=yes` and the daemon's stale-socket unlink on start are -a **pair**: preserving the directory without the unlink means `bind` fails with -`EADDRINUSE` after a restart. - -## Packaging - -Built by the workspace derivation and extracted as its own package -(`nix build .#swarm-controller`). Deliberately **not** in `nix/packages`' -`daemonBins` — that list is the core stack and drives the bundle -`services.hyperhive.c0re.package` points at, so folding this in would put a -swarm-scoped service into every hive's closure. diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs deleted file mode 100644 index aefb3a75..00000000 --- a/swarm-controller/src/main.rs +++ /dev/null @@ -1,117 +0,0 @@ -//! Swarm-level controller daemon. Runs as the unprivileged -//! `swarm-controller` user on whichever host the operator flips -//! `services.hyperhive.swarm.controller.enable` on, and serves HTTP over a -//! unix socket that the hive-gateway's nginx proxies to. -//! -//! **Today it serves one endpoint and owns no state.** That is deliberate: -//! this slice exists to make the *unit* real — service user, runtime and -//! state directories, socket, nginx reachability — so the swarm-level -//! surfaces that follow have somewhere to land. Guessing those surfaces -//! now would bake in a shape nobody has agreed to. -//! -//! Distinct from `hive-c0re`, which is per-hive: c0re owns the agents on -//! one host, this owns what is true across hives. - -use std::os::unix::fs::PermissionsExt as _; -use std::path::PathBuf; - -use anyhow::{Context, Result}; -use axum::{Router, routing::get}; - -/// Where the daemon binds, overridable via `SWARM_CONTROLLER_SOCKET`. -/// -/// A compiled-in default is legitimate here and is *not* the mistake that -/// a hardcoded remote address would be: this is a path this process -/// **creates**, not an address it hopes to find something at. systemd's -/// `RuntimeDirectory=swarm-controller` makes the parent exist before -/// `ExecStart`, so the default names a directory the unit just produced. -/// -/// The directory is its own — deliberately not shared with hive-c0re's -/// `/run/hyperhive`. nginx reaches a unix upstream by having the socket's -/// *directory* bind-mounted into the gateway container, so co-locating -/// this socket with c0re's admin socket would hand the gateway the admin -/// socket along with it. -const DEFAULT_SOCKET: &str = "/run/swarm-controller/controller.sock"; - -fn socket_path() -> PathBuf { - std::env::var_os("SWARM_CONTROLLER_SOCKET") - .map_or_else(|| PathBuf::from(DEFAULT_SOCKET), PathBuf::from) -} - -/// Liveness probe. Returns the build's version so an operator can tell -/// *which* controller answered without shelling onto the host. -async fn health() -> &'static str { - concat!("swarm-controller ", env!("CARGO_PKG_VERSION"), "\n") -} - -#[tokio::main] -async fn main() -> Result<()> { - tracing_subscriber::fmt() - .with_env_filter( - tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), - ) - .init(); - - let path = socket_path(); - - // `RuntimeDirectoryPreserve=yes` keeps the directory across a restart, - // so a socket file from the previous run can outlive the process that - // owned it and `bind` would fail with EADDRINUSE. Unlinking a stale - // socket is safe precisely because the directory is ours alone: nothing - // else can have put a file at this path. - if let Err(e) = std::fs::remove_file(&path) - && e.kind() != std::io::ErrorKind::NotFound - { - return Err(e).with_context(|| format!("clearing stale socket at {}", path.display())); - } - - let listener = tokio::net::UnixListener::bind(&path) - .with_context(|| format!("binding {}", path.display()))?; - - // `bind` leaves the socket 0755, and connecting needs write — the - // gateway's nginx is a different user, so it would be locked out. - // 0666 matches how hive-c0re publishes the per-agent sockets - // (`socket_server::start`), and rests on the same argument: **the - // containing directory is the access control, not the socket mode.** - // This directory holds one socket and is bind-mounted into exactly - // one container. That is also why it must not be shared with - // hive-c0re's `/run/hyperhive` — with a 0666 socket, a directory - // that carries more than it should is the whole vulnerability. - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o666)) - .with_context(|| format!("chmod {}", path.display()))?; - tracing::info!(socket = %path.display(), "swarm-controller listening"); - - let app = Router::new().route("/health", get(health)); - axum::serve(listener, app) - .await - .context("serving swarm-controller") -} - -#[cfg(test)] -mod tests { - use super::DEFAULT_SOCKET; - use std::path::Path; - - /// The socket must not share a directory with anything else, because - /// the socket is `0666` and the directory is therefore the only access - /// control it has. `/run/hyperhive` in particular holds hive-c0re's - /// **admin** socket, and nginx reaches a unix upstream by mounting the - /// socket's whole directory into the gateway container. - /// - /// A test rather than a comment: the failure this guards against is a - /// one-word edit that looks tidier and reads fine in review. - #[test] - fn socket_lives_in_its_own_runtime_dir() { - let parent = Path::new(DEFAULT_SOCKET) - .parent() - .expect("socket path has a parent directory"); - assert_eq!( - parent, - Path::new("/run/swarm-controller"), - "the socket's directory is its access control — moving it under a shared \ - directory (notably /run/hyperhive, which holds the host admin socket) \ - exposes everything else in that directory to the gateway container" - ); - } -}