Compare commits
11 changed files with 0 additions and 359 deletions
|
|
@ -141,13 +141,6 @@ hand-maintained per-file tree drifts out of sync with the code.
|
||||||
dependency-free wire types.
|
dependency-free wire types.
|
||||||
- **`hive-metric/`** — small CLI to push a single labeled metric to the
|
- **`hive-metric/`** — small CLI to push a single labeled metric to the
|
||||||
OTEL collector via the OpenTelemetry Rust SDK / OTLP HTTP exporter.
|
OTEL collector via the OpenTelemetry Rust SDK / OTLP HTTP exporter.
|
||||||
- **`swarm-controller/`** — swarm-level daemon, opt-in per host
|
|
||||||
(`services.hyperhive.swarm.controller.enable`). Where `hive-c0re` owns
|
|
||||||
the agents on **one** host, this owns what is true **across** hives; a
|
|
||||||
swarm runs one of them, so most hives leave it off. Serves HTTP over a
|
|
||||||
unix socket the gateway's nginx proxies to — ⚠️ **the socket's
|
|
||||||
directory is its access control**; the constraint that governs it is in
|
|
||||||
the crate's README, and a unit test pins the path.
|
|
||||||
|
|
||||||
### External dependencies with no directory here
|
### External dependencies with no directory here
|
||||||
|
|
||||||
|
|
|
||||||
11
Cargo.lock
generated
11
Cargo.lock
generated
|
|
@ -4421,17 +4421,6 @@ version = "2.6.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
|
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "swarm-controller"
|
|
||||||
version = "0.1.0"
|
|
||||||
dependencies = [
|
|
||||||
"anyhow",
|
|
||||||
"axum",
|
|
||||||
"tokio",
|
|
||||||
"tracing",
|
|
||||||
"tracing-subscriber",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "syn"
|
name = "syn"
|
||||||
version = "1.0.109"
|
version = "1.0.109"
|
||||||
|
|
|
||||||
|
|
@ -21,7 +21,6 @@ members = [
|
||||||
"hive-sock-client",
|
"hive-sock-client",
|
||||||
"hive-types",
|
"hive-types",
|
||||||
"hivectl",
|
"hivectl",
|
||||||
"swarm-controller",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[workspace.package]
|
[workspace.package]
|
||||||
|
|
|
||||||
|
|
@ -235,19 +235,6 @@ one store, because the receiver keys destinations by *agent* so a
|
||||||
migrating agent keeps one unbroken incremental chain. See
|
migrating agent keeps one unbroken incremental chain. See
|
||||||
[snapshot-store.md](snapshot-store.md).
|
[snapshot-store.md](snapshot-store.md).
|
||||||
|
|
||||||
## Swarm controller
|
|
||||||
|
|
||||||
`services.hyperhive.swarm.controller.enable` runs the `swarm-controller`
|
|
||||||
daemon on this host. **Off by default and deliberately not derived from
|
|
||||||
`services.hyperhive.enable`**: a swarm has one controller, so enabling it
|
|
||||||
is a statement about swarm topology, not about whether hyperhive is
|
|
||||||
installed. Every hive runs `hive-c0re` (the agents on that host); one
|
|
||||||
hive additionally runs this (what is true across hives).
|
|
||||||
|
|
||||||
What it serves, why it is a unix socket rather than a port, and the
|
|
||||||
socket-directory constraint that governs where `socketPath` may point:
|
|
||||||
[`swarm-controller/README.md`](../swarm-controller/README.md).
|
|
||||||
|
|
||||||
## Cross-references
|
## Cross-references
|
||||||
|
|
||||||
- `docs/snapshot-store.md` — the swarm's `btrfs receive` endpoint, and
|
- `docs/snapshot-store.md` — the swarm's `btrfs receive` endpoint, and
|
||||||
|
|
|
||||||
|
|
@ -142,9 +142,6 @@
|
||||||
agentBaseToplevel = lib.mkDefault self.packages.x86_64-linux.agent-base-toplevel;
|
agentBaseToplevel = lib.mkDefault self.packages.x86_64-linux.agent-base-toplevel;
|
||||||
managerToplevel = lib.mkDefault self.packages.x86_64-linux.ruth-toplevel;
|
managerToplevel = lib.mkDefault self.packages.x86_64-linux.ruth-toplevel;
|
||||||
};
|
};
|
||||||
services.hyperhive.swarm.controller.package =
|
|
||||||
lib.mkDefault
|
|
||||||
self.packages.${pkgs.stdenv.hostPlatform.system}.swarm-controller;
|
|
||||||
services.hyperhive.gateway.swaggerUiTheme =
|
services.hyperhive.gateway.swaggerUiTheme =
|
||||||
lib.mkDefault
|
lib.mkDefault
|
||||||
self.packages.${pkgs.stdenv.hostPlatform.system}.swagger-ui-theme;
|
self.packages.${pkgs.stdenv.hostPlatform.system}.swagger-ui-theme;
|
||||||
|
|
|
||||||
|
|
@ -21,7 +21,6 @@
|
||||||
./hive-priv.nix
|
./hive-priv.nix
|
||||||
./hive-tls.nix
|
./hive-tls.nix
|
||||||
./otel.nix
|
./otel.nix
|
||||||
./swarm-controller.nix
|
|
||||||
./swarm-snapshot-store.nix
|
./swarm-snapshot-store.nix
|
||||||
./swarm-wireguard.nix
|
./swarm-wireguard.nix
|
||||||
./swarm.nix
|
./swarm.nix
|
||||||
|
|
|
||||||
|
|
@ -1,117 +0,0 @@
|
||||||
# The swarm-level controller daemon. Per-host opt-in: a swarm has one
|
|
||||||
# controller, so most hives leave this off and point at the hive that
|
|
||||||
# runs it. Distinct from hive-c0re, which every hive runs — c0re owns
|
|
||||||
# the agents on one host, this owns what is true across hives.
|
|
||||||
#
|
|
||||||
# Serves HTTP over a unix socket rather than a TCP port: the gateway's
|
|
||||||
# nginx is the only intended client, it reaches the socket through a
|
|
||||||
# bind-mount, and a socket that is never bound to an address cannot be
|
|
||||||
# reached from off-host by mistake.
|
|
||||||
{
|
|
||||||
lib,
|
|
||||||
config,
|
|
||||||
...
|
|
||||||
}:
|
|
||||||
let
|
|
||||||
cfg = config.services.hyperhive.swarm.controller;
|
|
||||||
in
|
|
||||||
{
|
|
||||||
options.services.hyperhive.swarm.controller = {
|
|
||||||
enable = lib.mkOption {
|
|
||||||
type = lib.types.bool;
|
|
||||||
default = false;
|
|
||||||
description = ''
|
|
||||||
Run the swarm-controller daemon on this host. Off by default and
|
|
||||||
deliberately not derived from `services.hyperhive.enable`: a swarm
|
|
||||||
has one controller, so enabling it per hive is a decision about
|
|
||||||
swarm topology, not about whether hyperhive is installed.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
package = lib.mkOption {
|
|
||||||
type = lib.types.package;
|
|
||||||
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-controller";
|
|
||||||
description = ''
|
|
||||||
swarm-controller package. Wired by default from this flake's own
|
|
||||||
package set (see `flake.nix`); override to run a different build.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
socketPath = lib.mkOption {
|
|
||||||
type = lib.types.str;
|
|
||||||
default = "/run/swarm-controller/controller.sock";
|
|
||||||
description = ''
|
|
||||||
Unix socket the daemon serves on, and the path the gateway's nginx
|
|
||||||
proxies to.
|
|
||||||
|
|
||||||
The **directory** is the access control here, not the socket mode:
|
|
||||||
the socket itself is `0666` (nginx runs as another user, and
|
|
||||||
`connect(2)` needs write), exactly as hive-c0re publishes the
|
|
||||||
per-agent sockets. What keeps that safe is that the directory holds
|
|
||||||
one socket and is bind-mounted into one container. Moving this path
|
|
||||||
under a directory that carries anything else — `/run/hyperhive`,
|
|
||||||
which holds the host admin socket, above all — hands whatever else
|
|
||||||
lives there to every consumer that mounts it.
|
|
||||||
|
|
||||||
Changing this therefore means re-checking the gateway bind-mount,
|
|
||||||
not just the daemon.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) {
|
|
||||||
users.users.swarm-controller = {
|
|
||||||
isSystemUser = true;
|
|
||||||
group = "swarm-controller";
|
|
||||||
description = "hyperhive swarm-controller daemon";
|
|
||||||
};
|
|
||||||
users.groups.swarm-controller = { };
|
|
||||||
|
|
||||||
systemd.services.swarm-controller = {
|
|
||||||
description = "hyperhive swarm-level controller daemon";
|
|
||||||
wantedBy = [ "multi-user.target" ];
|
|
||||||
after = [ "network.target" ];
|
|
||||||
|
|
||||||
serviceConfig = {
|
|
||||||
ExecStart = "${cfg.package}/bin/swarm-controller";
|
|
||||||
User = "swarm-controller";
|
|
||||||
Group = "swarm-controller";
|
|
||||||
Restart = "on-failure";
|
|
||||||
RestartSec = "5s";
|
|
||||||
|
|
||||||
# `/run/swarm-controller` — its own directory, holding only the
|
|
||||||
# socket. See `socketPath`'s description for why that is a security
|
|
||||||
# property and not tidiness.
|
|
||||||
RuntimeDirectory = "swarm-controller";
|
|
||||||
# 0751: traverse-only for others, so the gateway's nginx can reach
|
|
||||||
# the socket path without being able to list the directory. Same
|
|
||||||
# shape (and same reason) as hive-c0re's runtime dir.
|
|
||||||
RuntimeDirectoryMode = "0751";
|
|
||||||
# Preserved across restarts so the bind-mount source never vanishes
|
|
||||||
# from under a running gateway container. The daemon unlinks a stale
|
|
||||||
# socket on start, which is what makes preservation safe.
|
|
||||||
RuntimeDirectoryPreserve = "yes";
|
|
||||||
|
|
||||||
StateDirectory = "swarm-controller";
|
|
||||||
StateDirectoryMode = "0750";
|
|
||||||
|
|
||||||
# Nothing here needs a writable filesystem, real privileges, or a
|
|
||||||
# view of the rest of the machine; the daemon reads its socket path
|
|
||||||
# from config and serves.
|
|
||||||
PrivateTmp = true;
|
|
||||||
ProtectSystem = "strict";
|
|
||||||
ProtectHome = true;
|
|
||||||
NoNewPrivileges = true;
|
|
||||||
PrivateDevices = true;
|
|
||||||
ProtectKernelTunables = true;
|
|
||||||
ProtectKernelModules = true;
|
|
||||||
ProtectControlGroups = true;
|
|
||||||
RestrictAddressFamilies = [
|
|
||||||
"AF_UNIX"
|
|
||||||
];
|
|
||||||
};
|
|
||||||
|
|
||||||
environment.SWARM_CONTROLLER_SOCKET = cfg.socketPath;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
@ -133,16 +133,6 @@ in
|
||||||
}
|
}
|
||||||
// binPkgs
|
// binPkgs
|
||||||
// {
|
// {
|
||||||
# Swarm-level controller daemon. Deliberately NOT in `daemonBins`:
|
|
||||||
# that list is the core stack — the binaries hive-c0re and the agent
|
|
||||||
# harness are made of — and it drives the `default` bundle that
|
|
||||||
# `services.hyperhive.c0re.package` points at. This daemon is a
|
|
||||||
# separate swarm-scoped service with its own module and its own
|
|
||||||
# `package` option, and one hive in a swarm runs it, so folding it
|
|
||||||
# into the core bundle would put it in every hive's closure to no end.
|
|
||||||
# Uses the same per-bin extractor, just bound on its own.
|
|
||||||
swarm-controller = mkBinPackage "swarm-controller" "hyperhive swarm-level controller daemon";
|
|
||||||
|
|
||||||
# Bundled browser assets — see ./frontend.nix. Output is
|
# Bundled browser assets — see ./frontend.nix. Output is
|
||||||
# $out/{dashboard,agent}/ which the Rust binaries serve via
|
# $out/{dashboard,agent}/ which the Rust binaries serve via
|
||||||
# tower_http::ServeDir.
|
# tower_http::ServeDir.
|
||||||
|
|
|
||||||
|
|
@ -1,19 +0,0 @@
|
||||||
[package]
|
|
||||||
name = "swarm-controller"
|
|
||||||
version.workspace = true
|
|
||||||
readme = "README.md"
|
|
||||||
edition.workspace = true
|
|
||||||
|
|
||||||
[[bin]]
|
|
||||||
name = "swarm-controller"
|
|
||||||
path = "src/main.rs"
|
|
||||||
|
|
||||||
[dependencies]
|
|
||||||
anyhow.workspace = true
|
|
||||||
axum.workspace = true
|
|
||||||
tokio.workspace = true
|
|
||||||
tracing.workspace = true
|
|
||||||
tracing-subscriber.workspace = true
|
|
||||||
|
|
||||||
[lints]
|
|
||||||
workspace = true
|
|
||||||
|
|
@ -1,60 +0,0 @@
|
||||||
# swarm-controller
|
|
||||||
|
|
||||||
The **swarm-level** daemon. Where `hive-c0re` owns the agents on one host, this
|
|
||||||
owns what is true *across* hives — so a swarm runs one of them and most hives
|
|
||||||
leave it off.
|
|
||||||
|
|
||||||
Opt-in per host via `services.hyperhive.swarm.controller.enable`, which is
|
|
||||||
deliberately **not** derived from `services.hyperhive.enable`: turning it on is
|
|
||||||
a statement about swarm topology, not about whether hyperhive is installed.
|
|
||||||
|
|
||||||
## What it does today
|
|
||||||
|
|
||||||
Serves one `/health` endpoint and holds no state.
|
|
||||||
|
|
||||||
That is the whole intent of the first slice. The point is to make the *unit*
|
|
||||||
real — service user, runtime and state directories, socket, nginx
|
|
||||||
reachability — so the swarm-level surfaces that follow have somewhere to land.
|
|
||||||
Inventing those surfaces before they are agreed would bake in a shape nobody
|
|
||||||
chose. See #3066 and the `hyperhive.swarm` consolidation epic.
|
|
||||||
|
|
||||||
## Why a unix socket, not a port
|
|
||||||
|
|
||||||
The hive-gateway's nginx is the only intended client and reaches the socket
|
|
||||||
through a bind-mount. A listener that is never bound to an address cannot be
|
|
||||||
reached from off-host by mistake.
|
|
||||||
|
|
||||||
The socket path is `services.hyperhive.swarm.controller.socketPath`, default
|
|
||||||
`/run/swarm-controller/controller.sock`, exported to the process as
|
|
||||||
`SWARM_CONTROLLER_SOCKET`.
|
|
||||||
|
|
||||||
## ⚠️ The socket's directory is its access control
|
|
||||||
|
|
||||||
The socket is `0666`. It has to be: nginx runs as a different user and
|
|
||||||
`connect(2)` needs write. This matches how `hive-c0re` publishes the per-agent
|
|
||||||
sockets, and rests on the same argument — *"the bind source dir is per-agent on
|
|
||||||
host so blast radius is unchanged."*
|
|
||||||
|
|
||||||
What keeps that safe is that the directory holds **one** socket and is
|
|
||||||
bind-mounted into **one** container. So:
|
|
||||||
|
|
||||||
> **Never point `socketPath` at a directory that carries anything else.**
|
|
||||||
> `/run/hyperhive` above all — it holds `host.sock`, the host **admin** socket.
|
|
||||||
> Mounting that directory to reach this socket would hand the gateway container
|
|
||||||
> the admin socket along with it.
|
|
||||||
|
|
||||||
Changing `socketPath` therefore means re-checking the gateway bind-mount, not
|
|
||||||
just the daemon. A unit test pins the default path so a tidying edit fails
|
|
||||||
instead of reviewing cleanly.
|
|
||||||
|
|
||||||
`RuntimeDirectoryPreserve=yes` and the daemon's stale-socket unlink on start are
|
|
||||||
a **pair**: preserving the directory without the unlink means `bind` fails with
|
|
||||||
`EADDRINUSE` after a restart.
|
|
||||||
|
|
||||||
## Packaging
|
|
||||||
|
|
||||||
Built by the workspace derivation and extracted as its own package
|
|
||||||
(`nix build .#swarm-controller`). Deliberately **not** in `nix/packages`'
|
|
||||||
`daemonBins` — that list is the core stack and drives the bundle
|
|
||||||
`services.hyperhive.c0re.package` points at, so folding this in would put a
|
|
||||||
swarm-scoped service into every hive's closure.
|
|
||||||
|
|
@ -1,117 +0,0 @@
|
||||||
//! Swarm-level controller daemon. Runs as the unprivileged
|
|
||||||
//! `swarm-controller` user on whichever host the operator flips
|
|
||||||
//! `services.hyperhive.swarm.controller.enable` on, and serves HTTP over a
|
|
||||||
//! unix socket that the hive-gateway's nginx proxies to.
|
|
||||||
//!
|
|
||||||
//! **Today it serves one endpoint and owns no state.** That is deliberate:
|
|
||||||
//! this slice exists to make the *unit* real — service user, runtime and
|
|
||||||
//! state directories, socket, nginx reachability — so the swarm-level
|
|
||||||
//! surfaces that follow have somewhere to land. Guessing those surfaces
|
|
||||||
//! now would bake in a shape nobody has agreed to.
|
|
||||||
//!
|
|
||||||
//! Distinct from `hive-c0re`, which is per-hive: c0re owns the agents on
|
|
||||||
//! one host, this owns what is true across hives.
|
|
||||||
|
|
||||||
use std::os::unix::fs::PermissionsExt as _;
|
|
||||||
use std::path::PathBuf;
|
|
||||||
|
|
||||||
use anyhow::{Context, Result};
|
|
||||||
use axum::{Router, routing::get};
|
|
||||||
|
|
||||||
/// Where the daemon binds, overridable via `SWARM_CONTROLLER_SOCKET`.
|
|
||||||
///
|
|
||||||
/// A compiled-in default is legitimate here and is *not* the mistake that
|
|
||||||
/// a hardcoded remote address would be: this is a path this process
|
|
||||||
/// **creates**, not an address it hopes to find something at. systemd's
|
|
||||||
/// `RuntimeDirectory=swarm-controller` makes the parent exist before
|
|
||||||
/// `ExecStart`, so the default names a directory the unit just produced.
|
|
||||||
///
|
|
||||||
/// The directory is its own — deliberately not shared with hive-c0re's
|
|
||||||
/// `/run/hyperhive`. nginx reaches a unix upstream by having the socket's
|
|
||||||
/// *directory* bind-mounted into the gateway container, so co-locating
|
|
||||||
/// this socket with c0re's admin socket would hand the gateway the admin
|
|
||||||
/// socket along with it.
|
|
||||||
const DEFAULT_SOCKET: &str = "/run/swarm-controller/controller.sock";
|
|
||||||
|
|
||||||
fn socket_path() -> PathBuf {
|
|
||||||
std::env::var_os("SWARM_CONTROLLER_SOCKET")
|
|
||||||
.map_or_else(|| PathBuf::from(DEFAULT_SOCKET), PathBuf::from)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Liveness probe. Returns the build's version so an operator can tell
|
|
||||||
/// *which* controller answered without shelling onto the host.
|
|
||||||
async fn health() -> &'static str {
|
|
||||||
concat!("swarm-controller ", env!("CARGO_PKG_VERSION"), "\n")
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::main]
|
|
||||||
async fn main() -> Result<()> {
|
|
||||||
tracing_subscriber::fmt()
|
|
||||||
.with_env_filter(
|
|
||||||
tracing_subscriber::EnvFilter::try_from_default_env()
|
|
||||||
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
|
|
||||||
)
|
|
||||||
.init();
|
|
||||||
|
|
||||||
let path = socket_path();
|
|
||||||
|
|
||||||
// `RuntimeDirectoryPreserve=yes` keeps the directory across a restart,
|
|
||||||
// so a socket file from the previous run can outlive the process that
|
|
||||||
// owned it and `bind` would fail with EADDRINUSE. Unlinking a stale
|
|
||||||
// socket is safe precisely because the directory is ours alone: nothing
|
|
||||||
// else can have put a file at this path.
|
|
||||||
if let Err(e) = std::fs::remove_file(&path)
|
|
||||||
&& e.kind() != std::io::ErrorKind::NotFound
|
|
||||||
{
|
|
||||||
return Err(e).with_context(|| format!("clearing stale socket at {}", path.display()));
|
|
||||||
}
|
|
||||||
|
|
||||||
let listener = tokio::net::UnixListener::bind(&path)
|
|
||||||
.with_context(|| format!("binding {}", path.display()))?;
|
|
||||||
|
|
||||||
// `bind` leaves the socket 0755, and connecting needs write — the
|
|
||||||
// gateway's nginx is a different user, so it would be locked out.
|
|
||||||
// 0666 matches how hive-c0re publishes the per-agent sockets
|
|
||||||
// (`socket_server::start`), and rests on the same argument: **the
|
|
||||||
// containing directory is the access control, not the socket mode.**
|
|
||||||
// This directory holds one socket and is bind-mounted into exactly
|
|
||||||
// one container. That is also why it must not be shared with
|
|
||||||
// hive-c0re's `/run/hyperhive` — with a 0666 socket, a directory
|
|
||||||
// that carries more than it should is the whole vulnerability.
|
|
||||||
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o666))
|
|
||||||
.with_context(|| format!("chmod {}", path.display()))?;
|
|
||||||
tracing::info!(socket = %path.display(), "swarm-controller listening");
|
|
||||||
|
|
||||||
let app = Router::new().route("/health", get(health));
|
|
||||||
axum::serve(listener, app)
|
|
||||||
.await
|
|
||||||
.context("serving swarm-controller")
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::DEFAULT_SOCKET;
|
|
||||||
use std::path::Path;
|
|
||||||
|
|
||||||
/// The socket must not share a directory with anything else, because
|
|
||||||
/// the socket is `0666` and the directory is therefore the only access
|
|
||||||
/// control it has. `/run/hyperhive` in particular holds hive-c0re's
|
|
||||||
/// **admin** socket, and nginx reaches a unix upstream by mounting the
|
|
||||||
/// socket's whole directory into the gateway container.
|
|
||||||
///
|
|
||||||
/// A test rather than a comment: the failure this guards against is a
|
|
||||||
/// one-word edit that looks tidier and reads fine in review.
|
|
||||||
#[test]
|
|
||||||
fn socket_lives_in_its_own_runtime_dir() {
|
|
||||||
let parent = Path::new(DEFAULT_SOCKET)
|
|
||||||
.parent()
|
|
||||||
.expect("socket path has a parent directory");
|
|
||||||
assert_eq!(
|
|
||||||
parent,
|
|
||||||
Path::new("/run/swarm-controller"),
|
|
||||||
"the socket's directory is its access control — moving it under a shared \
|
|
||||||
directory (notably /run/hyperhive, which holds the host admin socket) \
|
|
||||||
exposes everything else in that directory to the gateway container"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Loading…
Reference in a new issue