vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

Compare commits

base: vinzenz:dde0e1a8c103832aa388c3f972090dd7a723a740
Branches Tags
vinzenz:main
..
compare: vinzenz:5235166bb3e6fe7a63fc845a10ce28b95517e4f7
Branches Tags
vinzenz:main
11 changed files with 0 additions and 359 deletions
Download patch file Download diff file Expand all files Collapse all files

7
CLAUDE.md
View file

@ -141,13 +141,6 @@ hand-maintained per-file tree drifts out of sync with the code.
dependency-free wire types.
- **`hive-metric/`** — small CLI to push a single labeled metric to the
OTEL collector via the OpenTelemetry Rust SDK / OTLP HTTP exporter.
- **`swarm-controller/`** — swarm-level daemon, opt-in per host
(`services.hyperhive.swarm.controller.enable`). Where `hive-c0re` owns
the agents on **one** host, this owns what is true **across** hives; a
swarm runs one of them, so most hives leave it off. Serves HTTP over a
unix socket the gateway's nginx proxies to — ⚠️ **the socket's
directory is its access control**; the constraint that governs it is in
the crate's README, and a unit test pins the path.
### External dependencies with no directory here

11
Cargo.lock generated
View file

@ -4421,17 +4421,6 @@ version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "swarm-controller"
version = "0.1.0"
dependencies = [
"anyhow",
"axum",
"tokio",
"tracing",
"tracing-subscriber",
]
[[package]]
name = "syn"
version = "1.0.109"

1
Cargo.toml
View file

@ -21,7 +21,6 @@ members = [
"hive-sock-client",
"hive-types",
"hivectl",
"swarm-controller",
]
[workspace.package]

13
docs/swarm.md
View file

@ -235,19 +235,6 @@ one store, because the receiver keys destinations by *agent* so a
migrating agent keeps one unbroken incremental chain. See
[snapshot-store.md](snapshot-store.md).
## Swarm controller
`services.hyperhive.swarm.controller.enable` runs the `swarm-controller`
daemon on this host. **Off by default and deliberately not derived from
`services.hyperhive.enable`**: a swarm has one controller, so enabling it
is a statement about swarm topology, not about whether hyperhive is
installed. Every hive runs `hive-c0re` (the agents on that host); one
hive additionally runs this (what is true across hives).
What it serves, why it is a unix socket rather than a port, and the
socket-directory constraint that governs where `socketPath` may point:
[`swarm-controller/README.md`](../swarm-controller/README.md).
## Cross-references
- `docs/snapshot-store.md` — the swarm's `btrfs receive` endpoint, and

3
flake.nix
View file

@ -142,9 +142,6 @@
agentBaseToplevel = lib.mkDefault self.packages.x86_64-linux.agent-base-toplevel;
managerToplevel = lib.mkDefault self.packages.x86_64-linux.ruth-toplevel;
};
services.hyperhive.swarm.controller.package =
lib.mkDefault
self.packages.${pkgs.stdenv.hostPlatform.system}.swarm-controller;
services.hyperhive.gateway.swaggerUiTheme =
lib.mkDefault
self.packages.${pkgs.stdenv.hostPlatform.system}.swagger-ui-theme;

1
nix/host-modules/default.nix
View file

@ -21,7 +21,6 @@
./hive-priv.nix
./hive-tls.nix
./otel.nix
./swarm-controller.nix
./swarm-snapshot-store.nix
./swarm-wireguard.nix
./swarm.nix

117
nix/host-modules/swarm-controller.nix
View file

@ -1,117 +0,0 @@
# The swarm-level controller daemon. Per-host opt-in: a swarm has one
# controller, so most hives leave this off and point at the hive that
# runs it. Distinct from hive-c0re, which every hive runs — c0re owns
# the agents on one host, this owns what is true across hives.
#
# Serves HTTP over a unix socket rather than a TCP port: the gateway's
# nginx is the only intended client, it reaches the socket through a
# bind-mount, and a socket that is never bound to an address cannot be
# reached from off-host by mistake.
{
lib,
config,
...
}:
let
cfg = config.services.hyperhive.swarm.controller;
in
{
options.services.hyperhive.swarm.controller = {
enable = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Run the swarm-controller daemon on this host. Off by default and
deliberately not derived from `services.hyperhive.enable`: a swarm
has one controller, so enabling it per hive is a decision about
swarm topology, not about whether hyperhive is installed.
'';
};
package = lib.mkOption {
type = lib.types.package;
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-controller";
description = ''
swarm-controller package. Wired by default from this flake's own
package set (see `flake.nix`); override to run a different build.
'';
};
socketPath = lib.mkOption {
type = lib.types.str;
default = "/run/swarm-controller/controller.sock";
description = ''
Unix socket the daemon serves on, and the path the gateway's nginx
proxies to.
The **directory** is the access control here, not the socket mode:
the socket itself is `0666` (nginx runs as another user, and
`connect(2)` needs write), exactly as hive-c0re publishes the
per-agent sockets. What keeps that safe is that the directory holds
one socket and is bind-mounted into one container. Moving this path
under a directory that carries anything else — `/run/hyperhive`,
which holds the host admin socket, above all — hands whatever else
lives there to every consumer that mounts it.
Changing this therefore means re-checking the gateway bind-mount,
not just the daemon.
'';
};
};
config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) {
users.users.swarm-controller = {
isSystemUser = true;
group = "swarm-controller";
description = "hyperhive swarm-controller daemon";
};
users.groups.swarm-controller = { };
systemd.services.swarm-controller = {
description = "hyperhive swarm-level controller daemon";
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
serviceConfig = {
ExecStart = "${cfg.package}/bin/swarm-controller";
User = "swarm-controller";
Group = "swarm-controller";
Restart = "on-failure";
RestartSec = "5s";
# `/run/swarm-controller` — its own directory, holding only the
# socket. See `socketPath`'s description for why that is a security
# property and not tidiness.
RuntimeDirectory = "swarm-controller";
# 0751: traverse-only for others, so the gateway's nginx can reach
# the socket path without being able to list the directory. Same
# shape (and same reason) as hive-c0re's runtime dir.
RuntimeDirectoryMode = "0751";
# Preserved across restarts so the bind-mount source never vanishes
# from under a running gateway container. The daemon unlinks a stale
# socket on start, which is what makes preservation safe.
RuntimeDirectoryPreserve = "yes";
StateDirectory = "swarm-controller";
StateDirectoryMode = "0750";
# Nothing here needs a writable filesystem, real privileges, or a
# view of the rest of the machine; the daemon reads its socket path
# from config and serves.
PrivateTmp = true;
ProtectSystem = "strict";
ProtectHome = true;
NoNewPrivileges = true;
PrivateDevices = true;
ProtectKernelTunables = true;
ProtectKernelModules = true;
ProtectControlGroups = true;
RestrictAddressFamilies = [
"AF_UNIX"
];
};
environment.SWARM_CONTROLLER_SOCKET = cfg.socketPath;
};
};
}

10
nix/packages/default.nix
View file

@ -133,16 +133,6 @@ in
}
// binPkgs
// {
# Swarm-level controller daemon. Deliberately NOT in `daemonBins`:
# that list is the core stack — the binaries hive-c0re and the agent
# harness are made of — and it drives the `default` bundle that
# `services.hyperhive.c0re.package` points at. This daemon is a
# separate swarm-scoped service with its own module and its own
# `package` option, and one hive in a swarm runs it, so folding it
# into the core bundle would put it in every hive's closure to no end.
# Uses the same per-bin extractor, just bound on its own.
swarm-controller = mkBinPackage "swarm-controller" "hyperhive swarm-level controller daemon";
# Bundled browser assets — see ./frontend.nix. Output is
# $out/{dashboard,agent}/ which the Rust binaries serve via
# tower_http::ServeDir.

19
swarm-controller/Cargo.toml
View file

@ -1,19 +0,0 @@
[package]
name = "swarm-controller"
version.workspace = true
readme = "README.md"
edition.workspace = true
[[bin]]
name = "swarm-controller"
path = "src/main.rs"
[dependencies]
anyhow.workspace = true
axum.workspace = true
tokio.workspace = true
tracing.workspace = true
tracing-subscriber.workspace = true
[lints]
workspace = true

60
swarm-controller/README.md
View file

@ -1,60 +0,0 @@
# swarm-controller
The **swarm-level** daemon. Where `hive-c0re` owns the agents on one host, this
owns what is true *across* hives — so a swarm runs one of them and most hives
leave it off.
Opt-in per host via `services.hyperhive.swarm.controller.enable`, which is
deliberately **not** derived from `services.hyperhive.enable`: turning it on is
a statement about swarm topology, not about whether hyperhive is installed.
## What it does today
Serves one `/health` endpoint and holds no state.
That is the whole intent of the first slice. The point is to make the *unit*
real — service user, runtime and state directories, socket, nginx
reachability — so the swarm-level surfaces that follow have somewhere to land.
Inventing those surfaces before they are agreed would bake in a shape nobody
chose. See #3066 and the `hyperhive.swarm` consolidation epic.
## Why a unix socket, not a port
The hive-gateway's nginx is the only intended client and reaches the socket
through a bind-mount. A listener that is never bound to an address cannot be
reached from off-host by mistake.
The socket path is `services.hyperhive.swarm.controller.socketPath`, default
`/run/swarm-controller/controller.sock`, exported to the process as
`SWARM_CONTROLLER_SOCKET`.
## ⚠️ The socket's directory is its access control
The socket is `0666`. It has to be: nginx runs as a different user and
`connect(2)` needs write. This matches how `hive-c0re` publishes the per-agent
sockets, and rests on the same argument — *"the bind source dir is per-agent on
host so blast radius is unchanged."*
What keeps that safe is that the directory holds **one** socket and is
bind-mounted into **one** container. So:
> **Never point `socketPath` at a directory that carries anything else.**
> `/run/hyperhive` above all — it holds `host.sock`, the host **admin** socket.
> Mounting that directory to reach this socket would hand the gateway container
> the admin socket along with it.
Changing `socketPath` therefore means re-checking the gateway bind-mount, not
just the daemon. A unit test pins the default path so a tidying edit fails
instead of reviewing cleanly.
`RuntimeDirectoryPreserve=yes` and the daemon's stale-socket unlink on start are
a **pair**: preserving the directory without the unlink means `bind` fails with
`EADDRINUSE` after a restart.
## Packaging
Built by the workspace derivation and extracted as its own package
(`nix build .#swarm-controller`). Deliberately **not** in `nix/packages`'
`daemonBins` — that list is the core stack and drives the bundle
`services.hyperhive.c0re.package` points at, so folding this in would put a
swarm-scoped service into every hive's closure.

117
swarm-controller/src/main.rs
View file

@ -1,117 +0,0 @@
//! Swarm-level controller daemon. Runs as the unprivileged
//! `swarm-controller` user on whichever host the operator flips
//! `services.hyperhive.swarm.controller.enable` on, and serves HTTP over a
//! unix socket that the hive-gateway's nginx proxies to.
//!
//! **Today it serves one endpoint and owns no state.** That is deliberate:
//! this slice exists to make the *unit* real — service user, runtime and
//! state directories, socket, nginx reachability — so the swarm-level
//! surfaces that follow have somewhere to land. Guessing those surfaces
//! now would bake in a shape nobody has agreed to.
//!
//! Distinct from `hive-c0re`, which is per-hive: c0re owns the agents on
//! one host, this owns what is true across hives.
use std::os::unix::fs::PermissionsExt as _;
use std::path::PathBuf;
use anyhow::{Context, Result};
use axum::{Router, routing::get};
/// Where the daemon binds, overridable via `SWARM_CONTROLLER_SOCKET`.
///
/// A compiled-in default is legitimate here and is *not* the mistake that
/// a hardcoded remote address would be: this is a path this process
/// **creates**, not an address it hopes to find something at. systemd's
/// `RuntimeDirectory=swarm-controller` makes the parent exist before
/// `ExecStart`, so the default names a directory the unit just produced.
///
/// The directory is its own — deliberately not shared with hive-c0re's
/// `/run/hyperhive`. nginx reaches a unix upstream by having the socket's
/// *directory* bind-mounted into the gateway container, so co-locating
/// this socket with c0re's admin socket would hand the gateway the admin
/// socket along with it.
const DEFAULT_SOCKET: &str = "/run/swarm-controller/controller.sock";
fn socket_path() -> PathBuf {
std::env::var_os("SWARM_CONTROLLER_SOCKET")
.map_or_else(|| PathBuf::from(DEFAULT_SOCKET), PathBuf::from)
}
/// Liveness probe. Returns the build's version so an operator can tell
/// *which* controller answered without shelling onto the host.
async fn health() -> &'static str {
concat!("swarm-controller ", env!("CARGO_PKG_VERSION"), "\n")
}
#[tokio::main]
async fn main() -> Result<()> {
tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
)
.init();
let path = socket_path();
// `RuntimeDirectoryPreserve=yes` keeps the directory across a restart,
// so a socket file from the previous run can outlive the process that
// owned it and `bind` would fail with EADDRINUSE. Unlinking a stale
// socket is safe precisely because the directory is ours alone: nothing
// else can have put a file at this path.
if let Err(e) = std::fs::remove_file(&path)
&& e.kind() != std::io::ErrorKind::NotFound
{
return Err(e).with_context(|| format!("clearing stale socket at {}", path.display()));
}
let listener = tokio::net::UnixListener::bind(&path)
.with_context(|| format!("binding {}", path.display()))?;
// `bind` leaves the socket 0755, and connecting needs write — the
// gateway's nginx is a different user, so it would be locked out.
// 0666 matches how hive-c0re publishes the per-agent sockets
// (`socket_server::start`), and rests on the same argument: **the
// containing directory is the access control, not the socket mode.**
// This directory holds one socket and is bind-mounted into exactly
// one container. That is also why it must not be shared with
// hive-c0re's `/run/hyperhive` — with a 0666 socket, a directory
// that carries more than it should is the whole vulnerability.
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o666))
.with_context(|| format!("chmod {}", path.display()))?;
tracing::info!(socket = %path.display(), "swarm-controller listening");
let app = Router::new().route("/health", get(health));
axum::serve(listener, app)
.await
.context("serving swarm-controller")
}
#[cfg(test)]
mod tests {
use super::DEFAULT_SOCKET;
use std::path::Path;
/// The socket must not share a directory with anything else, because
/// the socket is `0666` and the directory is therefore the only access
/// control it has. `/run/hyperhive` in particular holds hive-c0re's
/// **admin** socket, and nginx reaches a unix upstream by mounting the
/// socket's whole directory into the gateway container.
///
/// A test rather than a comment: the failure this guards against is a
/// one-word edit that looks tidier and reads fine in review.
#[test]
fn socket_lives_in_its_own_runtime_dir() {
let parent = Path::new(DEFAULT_SOCKET)
.parent()
.expect("socket path has a parent directory");
assert_eq!(
parent,
Path::new("/run/swarm-controller"),
"the socket's directory is its access control — moving it under a shared \
directory (notably /run/hyperhive, which holds the host admin socket) \
exposes everything else in that directory to the gateway container"
);
}
}