Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
dde0e1a8c1 | ||
|
|
fb51006717 | ||
|
|
0fe2babbee | ||
|
|
435dfbfb33 | ||
|
|
f10f8a6bc6 | ||
|
|
898dde7402 |
11 changed files with 359 additions and 0 deletions
|
|
@ -141,6 +141,13 @@ hand-maintained per-file tree drifts out of sync with the code.
|
|||
dependency-free wire types.
|
||||
- **`hive-metric/`** — small CLI to push a single labeled metric to the
|
||||
OTEL collector via the OpenTelemetry Rust SDK / OTLP HTTP exporter.
|
||||
- **`swarm-controller/`** — swarm-level daemon, opt-in per host
|
||||
(`services.hyperhive.swarm.controller.enable`). Where `hive-c0re` owns
|
||||
the agents on **one** host, this owns what is true **across** hives; a
|
||||
swarm runs one of them, so most hives leave it off. Serves HTTP over a
|
||||
unix socket the gateway's nginx proxies to — ⚠️ **the socket's
|
||||
directory is its access control**; the constraint that governs it is in
|
||||
the crate's README, and a unit test pins the path.
|
||||
|
||||
### External dependencies with no directory here
|
||||
|
||||
|
|
|
|||
11
Cargo.lock
generated
11
Cargo.lock
generated
|
|
@ -4421,6 +4421,17 @@ version = "2.6.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
|
||||
|
||||
[[package]]
|
||||
name = "swarm-controller"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"axum",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.109"
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ members = [
|
|||
"hive-sock-client",
|
||||
"hive-types",
|
||||
"hivectl",
|
||||
"swarm-controller",
|
||||
]
|
||||
|
||||
[workspace.package]
|
||||
|
|
|
|||
|
|
@ -235,6 +235,19 @@ one store, because the receiver keys destinations by *agent* so a
|
|||
migrating agent keeps one unbroken incremental chain. See
|
||||
[snapshot-store.md](snapshot-store.md).
|
||||
|
||||
## Swarm controller
|
||||
|
||||
`services.hyperhive.swarm.controller.enable` runs the `swarm-controller`
|
||||
daemon on this host. **Off by default and deliberately not derived from
|
||||
`services.hyperhive.enable`**: a swarm has one controller, so enabling it
|
||||
is a statement about swarm topology, not about whether hyperhive is
|
||||
installed. Every hive runs `hive-c0re` (the agents on that host); one
|
||||
hive additionally runs this (what is true across hives).
|
||||
|
||||
What it serves, why it is a unix socket rather than a port, and the
|
||||
socket-directory constraint that governs where `socketPath` may point:
|
||||
[`swarm-controller/README.md`](../swarm-controller/README.md).
|
||||
|
||||
## Cross-references
|
||||
|
||||
- `docs/snapshot-store.md` — the swarm's `btrfs receive` endpoint, and
|
||||
|
|
|
|||
|
|
@ -142,6 +142,9 @@
|
|||
agentBaseToplevel = lib.mkDefault self.packages.x86_64-linux.agent-base-toplevel;
|
||||
managerToplevel = lib.mkDefault self.packages.x86_64-linux.ruth-toplevel;
|
||||
};
|
||||
services.hyperhive.swarm.controller.package =
|
||||
lib.mkDefault
|
||||
self.packages.${pkgs.stdenv.hostPlatform.system}.swarm-controller;
|
||||
services.hyperhive.gateway.swaggerUiTheme =
|
||||
lib.mkDefault
|
||||
self.packages.${pkgs.stdenv.hostPlatform.system}.swagger-ui-theme;
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@
|
|||
./hive-priv.nix
|
||||
./hive-tls.nix
|
||||
./otel.nix
|
||||
./swarm-controller.nix
|
||||
./swarm-snapshot-store.nix
|
||||
./swarm-wireguard.nix
|
||||
./swarm.nix
|
||||
|
|
|
|||
117
nix/host-modules/swarm-controller.nix
Normal file
117
nix/host-modules/swarm-controller.nix
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
# The swarm-level controller daemon. Per-host opt-in: a swarm has one
|
||||
# controller, so most hives leave this off and point at the hive that
|
||||
# runs it. Distinct from hive-c0re, which every hive runs — c0re owns
|
||||
# the agents on one host, this owns what is true across hives.
|
||||
#
|
||||
# Serves HTTP over a unix socket rather than a TCP port: the gateway's
|
||||
# nginx is the only intended client, it reaches the socket through a
|
||||
# bind-mount, and a socket that is never bound to an address cannot be
|
||||
# reached from off-host by mistake.
|
||||
{
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.services.hyperhive.swarm.controller;
|
||||
in
|
||||
{
|
||||
options.services.hyperhive.swarm.controller = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = ''
|
||||
Run the swarm-controller daemon on this host. Off by default and
|
||||
deliberately not derived from `services.hyperhive.enable`: a swarm
|
||||
has one controller, so enabling it per hive is a decision about
|
||||
swarm topology, not about whether hyperhive is installed.
|
||||
'';
|
||||
};
|
||||
|
||||
package = lib.mkOption {
|
||||
type = lib.types.package;
|
||||
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-controller";
|
||||
description = ''
|
||||
swarm-controller package. Wired by default from this flake's own
|
||||
package set (see `flake.nix`); override to run a different build.
|
||||
'';
|
||||
};
|
||||
|
||||
socketPath = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "/run/swarm-controller/controller.sock";
|
||||
description = ''
|
||||
Unix socket the daemon serves on, and the path the gateway's nginx
|
||||
proxies to.
|
||||
|
||||
The **directory** is the access control here, not the socket mode:
|
||||
the socket itself is `0666` (nginx runs as another user, and
|
||||
`connect(2)` needs write), exactly as hive-c0re publishes the
|
||||
per-agent sockets. What keeps that safe is that the directory holds
|
||||
one socket and is bind-mounted into one container. Moving this path
|
||||
under a directory that carries anything else — `/run/hyperhive`,
|
||||
which holds the host admin socket, above all — hands whatever else
|
||||
lives there to every consumer that mounts it.
|
||||
|
||||
Changing this therefore means re-checking the gateway bind-mount,
|
||||
not just the daemon.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) {
|
||||
users.users.swarm-controller = {
|
||||
isSystemUser = true;
|
||||
group = "swarm-controller";
|
||||
description = "hyperhive swarm-controller daemon";
|
||||
};
|
||||
users.groups.swarm-controller = { };
|
||||
|
||||
systemd.services.swarm-controller = {
|
||||
description = "hyperhive swarm-level controller daemon";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network.target" ];
|
||||
|
||||
serviceConfig = {
|
||||
ExecStart = "${cfg.package}/bin/swarm-controller";
|
||||
User = "swarm-controller";
|
||||
Group = "swarm-controller";
|
||||
Restart = "on-failure";
|
||||
RestartSec = "5s";
|
||||
|
||||
# `/run/swarm-controller` — its own directory, holding only the
|
||||
# socket. See `socketPath`'s description for why that is a security
|
||||
# property and not tidiness.
|
||||
RuntimeDirectory = "swarm-controller";
|
||||
# 0751: traverse-only for others, so the gateway's nginx can reach
|
||||
# the socket path without being able to list the directory. Same
|
||||
# shape (and same reason) as hive-c0re's runtime dir.
|
||||
RuntimeDirectoryMode = "0751";
|
||||
# Preserved across restarts so the bind-mount source never vanishes
|
||||
# from under a running gateway container. The daemon unlinks a stale
|
||||
# socket on start, which is what makes preservation safe.
|
||||
RuntimeDirectoryPreserve = "yes";
|
||||
|
||||
StateDirectory = "swarm-controller";
|
||||
StateDirectoryMode = "0750";
|
||||
|
||||
# Nothing here needs a writable filesystem, real privileges, or a
|
||||
# view of the rest of the machine; the daemon reads its socket path
|
||||
# from config and serves.
|
||||
PrivateTmp = true;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
NoNewPrivileges = true;
|
||||
PrivateDevices = true;
|
||||
ProtectKernelTunables = true;
|
||||
ProtectKernelModules = true;
|
||||
ProtectControlGroups = true;
|
||||
RestrictAddressFamilies = [
|
||||
"AF_UNIX"
|
||||
];
|
||||
};
|
||||
|
||||
environment.SWARM_CONTROLLER_SOCKET = cfg.socketPath;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -133,6 +133,16 @@ in
|
|||
}
|
||||
// binPkgs
|
||||
// {
|
||||
# Swarm-level controller daemon. Deliberately NOT in `daemonBins`:
|
||||
# that list is the core stack — the binaries hive-c0re and the agent
|
||||
# harness are made of — and it drives the `default` bundle that
|
||||
# `services.hyperhive.c0re.package` points at. This daemon is a
|
||||
# separate swarm-scoped service with its own module and its own
|
||||
# `package` option, and one hive in a swarm runs it, so folding it
|
||||
# into the core bundle would put it in every hive's closure to no end.
|
||||
# Uses the same per-bin extractor, just bound on its own.
|
||||
swarm-controller = mkBinPackage "swarm-controller" "hyperhive swarm-level controller daemon";
|
||||
|
||||
# Bundled browser assets — see ./frontend.nix. Output is
|
||||
# $out/{dashboard,agent}/ which the Rust binaries serve via
|
||||
# tower_http::ServeDir.
|
||||
|
|
|
|||
19
swarm-controller/Cargo.toml
Normal file
19
swarm-controller/Cargo.toml
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
[package]
|
||||
name = "swarm-controller"
|
||||
version.workspace = true
|
||||
readme = "README.md"
|
||||
edition.workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "swarm-controller"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
axum.workspace = true
|
||||
tokio.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
60
swarm-controller/README.md
Normal file
60
swarm-controller/README.md
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
# swarm-controller
|
||||
|
||||
The **swarm-level** daemon. Where `hive-c0re` owns the agents on one host, this
|
||||
owns what is true *across* hives — so a swarm runs one of them and most hives
|
||||
leave it off.
|
||||
|
||||
Opt-in per host via `services.hyperhive.swarm.controller.enable`, which is
|
||||
deliberately **not** derived from `services.hyperhive.enable`: turning it on is
|
||||
a statement about swarm topology, not about whether hyperhive is installed.
|
||||
|
||||
## What it does today
|
||||
|
||||
Serves one `/health` endpoint and holds no state.
|
||||
|
||||
That is the whole intent of the first slice. The point is to make the *unit*
|
||||
real — service user, runtime and state directories, socket, nginx
|
||||
reachability — so the swarm-level surfaces that follow have somewhere to land.
|
||||
Inventing those surfaces before they are agreed would bake in a shape nobody
|
||||
chose. See #3066 and the `hyperhive.swarm` consolidation epic.
|
||||
|
||||
## Why a unix socket, not a port
|
||||
|
||||
The hive-gateway's nginx is the only intended client and reaches the socket
|
||||
through a bind-mount. A listener that is never bound to an address cannot be
|
||||
reached from off-host by mistake.
|
||||
|
||||
The socket path is `services.hyperhive.swarm.controller.socketPath`, default
|
||||
`/run/swarm-controller/controller.sock`, exported to the process as
|
||||
`SWARM_CONTROLLER_SOCKET`.
|
||||
|
||||
## ⚠️ The socket's directory is its access control
|
||||
|
||||
The socket is `0666`. It has to be: nginx runs as a different user and
|
||||
`connect(2)` needs write. This matches how `hive-c0re` publishes the per-agent
|
||||
sockets, and rests on the same argument — *"the bind source dir is per-agent on
|
||||
host so blast radius is unchanged."*
|
||||
|
||||
What keeps that safe is that the directory holds **one** socket and is
|
||||
bind-mounted into **one** container. So:
|
||||
|
||||
> **Never point `socketPath` at a directory that carries anything else.**
|
||||
> `/run/hyperhive` above all — it holds `host.sock`, the host **admin** socket.
|
||||
> Mounting that directory to reach this socket would hand the gateway container
|
||||
> the admin socket along with it.
|
||||
|
||||
Changing `socketPath` therefore means re-checking the gateway bind-mount, not
|
||||
just the daemon. A unit test pins the default path so a tidying edit fails
|
||||
instead of reviewing cleanly.
|
||||
|
||||
`RuntimeDirectoryPreserve=yes` and the daemon's stale-socket unlink on start are
|
||||
a **pair**: preserving the directory without the unlink means `bind` fails with
|
||||
`EADDRINUSE` after a restart.
|
||||
|
||||
## Packaging
|
||||
|
||||
Built by the workspace derivation and extracted as its own package
|
||||
(`nix build .#swarm-controller`). Deliberately **not** in `nix/packages`'
|
||||
`daemonBins` — that list is the core stack and drives the bundle
|
||||
`services.hyperhive.c0re.package` points at, so folding this in would put a
|
||||
swarm-scoped service into every hive's closure.
|
||||
117
swarm-controller/src/main.rs
Normal file
117
swarm-controller/src/main.rs
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
//! Swarm-level controller daemon. Runs as the unprivileged
|
||||
//! `swarm-controller` user on whichever host the operator flips
|
||||
//! `services.hyperhive.swarm.controller.enable` on, and serves HTTP over a
|
||||
//! unix socket that the hive-gateway's nginx proxies to.
|
||||
//!
|
||||
//! **Today it serves one endpoint and owns no state.** That is deliberate:
|
||||
//! this slice exists to make the *unit* real — service user, runtime and
|
||||
//! state directories, socket, nginx reachability — so the swarm-level
|
||||
//! surfaces that follow have somewhere to land. Guessing those surfaces
|
||||
//! now would bake in a shape nobody has agreed to.
|
||||
//!
|
||||
//! Distinct from `hive-c0re`, which is per-hive: c0re owns the agents on
|
||||
//! one host, this owns what is true across hives.
|
||||
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use axum::{Router, routing::get};
|
||||
|
||||
/// Where the daemon binds, overridable via `SWARM_CONTROLLER_SOCKET`.
|
||||
///
|
||||
/// A compiled-in default is legitimate here and is *not* the mistake that
|
||||
/// a hardcoded remote address would be: this is a path this process
|
||||
/// **creates**, not an address it hopes to find something at. systemd's
|
||||
/// `RuntimeDirectory=swarm-controller` makes the parent exist before
|
||||
/// `ExecStart`, so the default names a directory the unit just produced.
|
||||
///
|
||||
/// The directory is its own — deliberately not shared with hive-c0re's
|
||||
/// `/run/hyperhive`. nginx reaches a unix upstream by having the socket's
|
||||
/// *directory* bind-mounted into the gateway container, so co-locating
|
||||
/// this socket with c0re's admin socket would hand the gateway the admin
|
||||
/// socket along with it.
|
||||
const DEFAULT_SOCKET: &str = "/run/swarm-controller/controller.sock";
|
||||
|
||||
fn socket_path() -> PathBuf {
|
||||
std::env::var_os("SWARM_CONTROLLER_SOCKET")
|
||||
.map_or_else(|| PathBuf::from(DEFAULT_SOCKET), PathBuf::from)
|
||||
}
|
||||
|
||||
/// Liveness probe. Returns the build's version so an operator can tell
|
||||
/// *which* controller answered without shelling onto the host.
|
||||
async fn health() -> &'static str {
|
||||
concat!("swarm-controller ", env!("CARGO_PKG_VERSION"), "\n")
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
tracing_subscriber::EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
|
||||
)
|
||||
.init();
|
||||
|
||||
let path = socket_path();
|
||||
|
||||
// `RuntimeDirectoryPreserve=yes` keeps the directory across a restart,
|
||||
// so a socket file from the previous run can outlive the process that
|
||||
// owned it and `bind` would fail with EADDRINUSE. Unlinking a stale
|
||||
// socket is safe precisely because the directory is ours alone: nothing
|
||||
// else can have put a file at this path.
|
||||
if let Err(e) = std::fs::remove_file(&path)
|
||||
&& e.kind() != std::io::ErrorKind::NotFound
|
||||
{
|
||||
return Err(e).with_context(|| format!("clearing stale socket at {}", path.display()));
|
||||
}
|
||||
|
||||
let listener = tokio::net::UnixListener::bind(&path)
|
||||
.with_context(|| format!("binding {}", path.display()))?;
|
||||
|
||||
// `bind` leaves the socket 0755, and connecting needs write — the
|
||||
// gateway's nginx is a different user, so it would be locked out.
|
||||
// 0666 matches how hive-c0re publishes the per-agent sockets
|
||||
// (`socket_server::start`), and rests on the same argument: **the
|
||||
// containing directory is the access control, not the socket mode.**
|
||||
// This directory holds one socket and is bind-mounted into exactly
|
||||
// one container. That is also why it must not be shared with
|
||||
// hive-c0re's `/run/hyperhive` — with a 0666 socket, a directory
|
||||
// that carries more than it should is the whole vulnerability.
|
||||
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o666))
|
||||
.with_context(|| format!("chmod {}", path.display()))?;
|
||||
tracing::info!(socket = %path.display(), "swarm-controller listening");
|
||||
|
||||
let app = Router::new().route("/health", get(health));
|
||||
axum::serve(listener, app)
|
||||
.await
|
||||
.context("serving swarm-controller")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::DEFAULT_SOCKET;
|
||||
use std::path::Path;
|
||||
|
||||
/// The socket must not share a directory with anything else, because
|
||||
/// the socket is `0666` and the directory is therefore the only access
|
||||
/// control it has. `/run/hyperhive` in particular holds hive-c0re's
|
||||
/// **admin** socket, and nginx reaches a unix upstream by mounting the
|
||||
/// socket's whole directory into the gateway container.
|
||||
///
|
||||
/// A test rather than a comment: the failure this guards against is a
|
||||
/// one-word edit that looks tidier and reads fine in review.
|
||||
#[test]
|
||||
fn socket_lives_in_its_own_runtime_dir() {
|
||||
let parent = Path::new(DEFAULT_SOCKET)
|
||||
.parent()
|
||||
.expect("socket path has a parent directory");
|
||||
assert_eq!(
|
||||
parent,
|
||||
Path::new("/run/swarm-controller"),
|
||||
"the socket's directory is its access control — moving it under a shared \
|
||||
directory (notably /run/hyperhive, which holds the host admin socket) \
|
||||
exposes everything else in that directory to the gateway container"
|
||||
);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue