c67100588e
Follow-up to the plumbing removal: docs/swarm/README.md gets the biggest rewrite (drops the whole "Fingerprint format" section, fixes the runtime-effects list, the WireGuard config example + "what the mesh does" bullet), docs/gateway.md and hive-gateway/options.nix drop 4 "needs no certFingerprint" mentions, swarm-peers-removed.nix's migration-warning text no longer tells an upgrading operator to carry a field over that no longer exists, swarm.nix/swarm-wireguard.nix/ swarm-controller.nix/swarm-controller's main.rs get comment fixes where they described the now-removed HYPERHIVE_PEERS shape. Also caught one more stale "peer hives" mention in docs/web-ui/README.md's SW4RM tab description that the first pass on this issue missed.
329 lines
14 KiB
Nix
329 lines
14 KiB
Nix
# The swarm-level controller daemon. Per-host opt-in: a swarm has one
|
|
# controller, so most hives leave this off and point at the hive that
|
|
# runs it. Distinct from hive-c0re, which every hive runs — c0re owns
|
|
# the agents on one host, this owns what is true across hives.
|
|
#
|
|
# Serves HTTP over a unix socket rather than a TCP port: the gateway's
|
|
# nginx is the only intended client, it runs on this same host and so
|
|
# reaches the socket by path, and a socket that is never bound to an
|
|
# address cannot be reached from off-host by mistake.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.swarm.controller;
|
|
autheliaCfg = config.services.hyperhive.swarm.authelia;
|
|
|
|
# What `swarmctl` needs in order to act on authelia from the host.
|
|
#
|
|
# Only set when authelia actually runs **here**: the controller can be
|
|
# enabled on a host that is not the swarm's SSO host, and in that case
|
|
# the right behaviour is for `swarmctl user add` to fail saying the
|
|
# value is unset. A guessed path would resolve cleanly and write a file
|
|
# nothing reads, which is the failure mode that costs an afternoon.
|
|
autheliaEnv = lib.optionalAttrs autheliaCfg.enable {
|
|
# The CONFIGURED authelia, not whatever is on PATH: the argon2
|
|
# parameters baked into a hash have to match the verifier's.
|
|
SWARMCTL_AUTHELIA_BIN = "${autheliaCfg.package}/bin/authelia";
|
|
SWARMCTL_AUTHELIA_USERS_FILE = autheliaCfg.hostUsersFile;
|
|
SWARMCTL_AUTHELIA_MACHINE = autheliaCfg.machine;
|
|
SWARMCTL_AUTHELIA_UNIT = autheliaCfg.unit;
|
|
};
|
|
|
|
natsCfg = config.services.hyperhive.swarm.nats;
|
|
|
|
# The controller's own OAuth2 client. It is NOT a hive: the per-hive
|
|
# clients the roster issues belong to hives, and the responder's client
|
|
# belongs to the responder. One identity per principal — the rule is that
|
|
# a principal's credentials all derive from the same identity, not that
|
|
# the swarm has one.
|
|
queueClientId = "swarm-controller";
|
|
|
|
# Both halves have to be here: authelia to have minted the secret, and the
|
|
# queue to connect to. Same guard, and the same reasoning, as `autheliaEnv`
|
|
# above — a value set on a host that runs neither would point at a file
|
|
# that does not exist and produce a daemon that retries forever.
|
|
queueLocal = autheliaCfg.enable && natsCfg.enable;
|
|
|
|
# `LoadCredential` and not a copy-oneshot, which is where this deliberately
|
|
# differs from the callout responder: that one delivers INTO a container,
|
|
# so it has to copy across a filesystem boundary. The controller is a plain
|
|
# host unit, so systemd can hand it the file directly — fewer moving parts,
|
|
# and the secret never gains a second on-disk copy to forget about.
|
|
queueEnv = lib.optionalAttrs queueLocal {
|
|
# The queue container shares the host netns, so loopback is correct here
|
|
# and is not the `localhost`-means-the-wrong-thing trap that applies
|
|
# inside agent containers.
|
|
SWARM_CONTROLLER_NATS_URL = "nats://127.0.0.1:${toString natsCfg.port}";
|
|
SWARM_CONTROLLER_OIDC_TOKEN_ENDPOINT = "${autheliaCfg.url}/api/oidc/token";
|
|
SWARM_CONTROLLER_OIDC_CLIENT_ID = queueClientId;
|
|
# `%d` is systemd's credentials directory: root reads the plaintext at
|
|
# unit start and the daemon's own user sees it 0400, without the unit
|
|
# ever being able to read the rest of authelia's state dir.
|
|
SWARM_CONTROLLER_OIDC_CLIENT_SECRET_FILE = "%d/queue-client.secret";
|
|
};
|
|
|
|
# Wrapped rather than documented: every one of these values is derived
|
|
# from an option this deployment already set, so making the operator
|
|
# re-supply them on the command line would be asking them to repeat the
|
|
# config back to it — and to get it wrong the day one of them changes.
|
|
swarmctlConfigured = pkgs.symlinkJoin {
|
|
name = "swarmctl-configured";
|
|
paths = [ cfg.swarmctlPackage ];
|
|
nativeBuildInputs = [ pkgs.makeWrapper ];
|
|
postBuild = ''
|
|
wrapProgram $out/bin/swarmctl ${
|
|
lib.concatStringsSep " " (
|
|
lib.mapAttrsToList (name: value: "--set ${name} ${lib.escapeShellArg value}") autheliaEnv
|
|
)
|
|
}
|
|
'';
|
|
};
|
|
in
|
|
{
|
|
options.services.hyperhive.swarm.controller = {
|
|
enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
description = ''
|
|
Run the swarm-controller daemon on this host. Off by default and
|
|
deliberately not derived from `services.hyperhive.enable`: a swarm
|
|
has one controller, so enabling it per hive is a decision about
|
|
swarm topology, not about whether hyperhive is installed.
|
|
|
|
`services.hyperhive.enableAllLocalDefaults` does assert it, and
|
|
that is not an exception to the rule above — it is the rule
|
|
applied. That mode says "this box is the whole deployment", which
|
|
answers the topology question outright, where
|
|
`services.hyperhive.enable` alone never can.
|
|
'';
|
|
};
|
|
|
|
package = lib.mkOption {
|
|
type = lib.types.package;
|
|
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-controller";
|
|
description = ''
|
|
swarm-controller package. Wired by default from this flake's own
|
|
package set (see `flake.nix`); override to run a different build.
|
|
'';
|
|
};
|
|
|
|
swarmctlPackage = lib.mkOption {
|
|
type = lib.types.package;
|
|
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarmctl";
|
|
description = ''
|
|
`swarmctl` package — the swarm operator's CLI, installed on this
|
|
host alongside the daemon and wrapped with the paths it needs.
|
|
|
|
A separate option from `package` rather than a second binary in
|
|
the same derivation: the CLI runs as root and acts directly,
|
|
the daemon runs unprivileged and serves a socket, and pinning one
|
|
without the other is a thing an operator may legitimately want.
|
|
'';
|
|
};
|
|
|
|
socketPath = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "/run/swarm-controller/controller.sock";
|
|
description = ''
|
|
Unix socket the daemon serves on, and the path the gateway's nginx
|
|
proxies to.
|
|
|
|
The **directory** is the access control here, not the socket mode:
|
|
the socket itself is `0666` (nginx runs as another user, and
|
|
`connect(2)` needs write), exactly as hive-c0re publishes the
|
|
per-agent sockets. What keeps that safe is that the directory holds
|
|
exactly one socket and is traverse-only (`0751`) for everyone else.
|
|
Moving this path under a directory that carries anything else —
|
|
`/run/hyperhive`, which holds the host admin socket, above all —
|
|
exposes whatever else lives there to the same reachability.
|
|
|
|
Changing this therefore means re-checking what else lives in the
|
|
new directory, not just the daemon.
|
|
'';
|
|
};
|
|
|
|
staleAfterSeconds = lib.mkOption {
|
|
type = lib.types.ints.positive;
|
|
default = 120;
|
|
description = ''
|
|
How old a hive's last status snapshot may be before
|
|
`GET /api/hives/status` reports it as `stale` rather than `fresh`.
|
|
|
|
This is a statement about how often hives *publish*, not about how
|
|
patient a reader is — set it above the publishing cadence or every
|
|
hive reads stale between offers. It is an option and not a
|
|
constant precisely because that cadence is a property of the
|
|
deployment.
|
|
|
|
Freshness is derived when the endpoint is read, never stored, so
|
|
changing this takes effect for the next request; no hive has to
|
|
re-publish anything.
|
|
'';
|
|
};
|
|
|
|
links = lib.mkOption {
|
|
type = lib.types.listOf (
|
|
lib.types.submodule {
|
|
options = {
|
|
label = lib.mkOption {
|
|
type = lib.types.str;
|
|
description = "Display label for the link.";
|
|
};
|
|
icon = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "";
|
|
description = "Optional icon emoji or short glyph.";
|
|
};
|
|
url = lib.mkOption {
|
|
type = lib.types.str;
|
|
description = "Full URL.";
|
|
};
|
|
};
|
|
}
|
|
);
|
|
default = [ ];
|
|
example = lib.literalExpression ''
|
|
[ { label = "Wiki"; icon = "📖"; url = "https://wiki.example.com/"; } ]
|
|
'';
|
|
description = ''
|
|
Quick links to swarm-wide services, surfaced by the swarm UI's
|
|
links menu (`GET /api/links`). Same shape and same
|
|
zero-code-change-to-extend idea as `hyperhive.dashboardLinks`
|
|
(`nix/agent-modules/dashboard-links.nix`), one level up: rather
|
|
than one central hardcoded list, each service's own module
|
|
contributes its own entry when it is actually enabled on this
|
|
host — `swarm-authelia.nix`, `hive-matrix.nix` and
|
|
`hive-forge/default.nix` all do — the same list-merge idiom
|
|
`services.hyperhive.gateway.localNames` already uses. A future
|
|
service module can push its own entry the same way, and an
|
|
operator can add arbitrary extra entries here directly; neither
|
|
needs a swarm-controller or swarm-ui change.
|
|
|
|
Only meaningful on the host that actually runs the controller —
|
|
entries contributed on any other host are computed but never
|
|
read. In a swarm that splits `swarm-authelia`/`hive-matrix`/
|
|
`hive-forge` across hosts other than the controller's, this list
|
|
only reflects what is enabled locally; see each contributing
|
|
module's own activation condition.
|
|
'';
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) {
|
|
users.users.swarm-controller = {
|
|
isSystemUser = true;
|
|
group = "swarm-controller";
|
|
description = "hyperhive swarm-controller daemon";
|
|
};
|
|
users.groups.swarm-controller = { };
|
|
|
|
# Installed host-wide, not into the daemon's unit: `swarmctl` is run
|
|
# by a human on this box and acts as root, so the daemon's sandbox is
|
|
# exactly what it must not inherit.
|
|
environment.systemPackages = [ swarmctlConfigured ];
|
|
|
|
# One declaration, two readers — the controller knows which client id it
|
|
# authenticates under, so making the operator restate it in authelia's
|
|
# client list would be a second source of truth for a string whose
|
|
# mismatch is an opaque 401 from the token endpoint. Same shape as the
|
|
# queue's own client declaration.
|
|
services.hyperhive.swarm.authelia.oidc.clients = lib.mkIf autheliaCfg.enable [
|
|
{
|
|
id = queueClientId;
|
|
description = "HyperHive swarm controller";
|
|
# `client_credentials`: a daemon authenticating as itself, with
|
|
# nobody to redirect. Declared rather than inferred from an empty
|
|
# redirect list, because authelia permits only the grants a client
|
|
# names and an omitted `grant_types` means authorization-code alone.
|
|
kind = "machine";
|
|
}
|
|
];
|
|
|
|
systemd.services.swarm-controller = {
|
|
description = "hyperhive swarm-level controller daemon";
|
|
wantedBy = [ "multi-user.target" ];
|
|
after = [ "network.target" ];
|
|
|
|
serviceConfig = {
|
|
ExecStart = "${cfg.package}/bin/swarm-controller";
|
|
|
|
# Only when the queue is actually reachable from here. An absent
|
|
# credential is not a failure: the daemon logs that no queue is
|
|
# configured and serves its HTTP surface, which is the correct
|
|
# behaviour on the hosts that do not run one.
|
|
LoadCredential = lib.mkIf queueLocal [
|
|
"queue-client.secret:${autheliaCfg.hostClientSecretDir}/${queueClientId}.secret"
|
|
];
|
|
User = "swarm-controller";
|
|
Group = "swarm-controller";
|
|
Restart = "on-failure";
|
|
RestartSec = "5s";
|
|
|
|
# `/run/swarm-controller` — its own directory, holding only the
|
|
# socket. See `socketPath`'s description for why that is a security
|
|
# property and not tidiness.
|
|
RuntimeDirectory = "swarm-controller";
|
|
# 0751: traverse-only for others, so the gateway's nginx can reach
|
|
# the socket path without being able to list the directory. Same
|
|
# shape (and same reason) as hive-c0re's runtime dir.
|
|
RuntimeDirectoryMode = "0751";
|
|
# Preserved across restarts so the path never vanishes from under a
|
|
# running nginx. The daemon unlinks a stale socket on start, which
|
|
# is what makes preservation safe.
|
|
RuntimeDirectoryPreserve = "yes";
|
|
|
|
StateDirectory = "swarm-controller";
|
|
StateDirectoryMode = "0750";
|
|
|
|
# Nothing here needs a writable filesystem, real privileges, or a
|
|
# view of the rest of the machine; the daemon reads its socket path
|
|
# from config and serves.
|
|
PrivateTmp = true;
|
|
ProtectSystem = "strict";
|
|
ProtectHome = true;
|
|
NoNewPrivileges = true;
|
|
PrivateDevices = true;
|
|
ProtectKernelTunables = true;
|
|
ProtectKernelModules = true;
|
|
ProtectControlGroups = true;
|
|
RestrictAddressFamilies = [
|
|
"AF_UNIX"
|
|
];
|
|
};
|
|
|
|
# Queue coordinates (`queueEnv`) merge in last and are present only
|
|
# where the queue and its IdP both run. The daemon refuses a PARTIAL
|
|
# set rather than treating it as absent, which is why they are built
|
|
# as one attrset and never assigned individually.
|
|
environment = {
|
|
SWARM_CONTROLLER_SOCKET = cfg.socketPath;
|
|
# The swarm's hive directory, JSON-encoded — the full directory
|
|
# (this daemon has no "self" hive to exclude, unlike
|
|
# `swarm.peerHives`, `swarm.hives` minus this hive) rather than
|
|
# peers-minus-self. Consumed by `GET /api/hives`
|
|
# (swarm-controller/src/main.rs::load_hives).
|
|
SWARM_CONTROLLER_HIVES = builtins.toJSON (
|
|
lib.mapAttrsToList (name: h: {
|
|
inherit name;
|
|
inherit (h) domain;
|
|
}) config.services.hyperhive.swarm.hives
|
|
);
|
|
# The merged links list — see `links`' description above for who
|
|
# contributes to it. Consumed by `GET /api/links`
|
|
# (swarm-controller/src/main.rs::load_links).
|
|
SWARM_CONTROLLER_LINKS = builtins.toJSON cfg.links;
|
|
# Staleness threshold for `GET /api/hives/status` — see
|
|
# `staleAfterSeconds`' description. Set unconditionally rather
|
|
# than inside `queueEnv`: it is not a queue coordinate, and
|
|
# nothing about it is unsafe to define on a host whose queue is
|
|
# off (the daemon just has nothing to apply it to).
|
|
SWARM_CONTROLLER_STALE_AFTER_SECS = toString cfg.staleAfterSeconds;
|
|
}
|
|
// queueEnv;
|
|
};
|
|
};
|
|
}
|