Watch
0
0
Fork
You've already forked hyperhive
0
hyperhive/nix/host-modules/swarm-controller.nix

1213 lines
54 KiB
Nix

# The swarm-level controller daemon. Per-host opt-in: a swarm has one
# controller, so most hives leave this off and point at the hive that
# runs it. Distinct from hive-c0re, which every hive runs — c0re owns
# the agents on one host, this owns what is true across hives.
#
# Serves HTTP over a unix socket rather than a TCP port: the gateway's
# nginx is the only intended client, it runs on this same host and so
# reaches the socket by path, and a socket that is never bound to an
# address cannot be reached from off-host by mistake.
#
# `swarm.controller.tcp.enable` adds a TLS listener beside the socket, which the
# gateway then dials instead of the socket. The controller checks nobody per
# request, so the listener completes a handshake for one identity only: a
# client certificate the swarm's store issued for the gateway, with the
# `clientAuth` usage. ./swarm-ui.nix still declares the vhost on the host that
# runs this daemon, so the gateway stays on that host.
{
pkgs,
lib,
config,
...
}:
let
cfg = config.services.hyperhive.swarm.controller;
deployCfg = config.services.hyperhive.deploy;
autheliaCfg = config.services.hyperhive.swarm.authelia;
natsCfg = config.services.hyperhive.swarm.nats;
swarmCfg = config.services.hyperhive.swarm;
# Who may connect to the controller's socket: nginx (the gateway's `/api/`
# and `/webhook/forge/` locations) and root (`swarmctl`). A group of its own
# rather than `swarm-controller`: that group also owns the state directory,
# and nginx has no business reading it.
clientsGroup = "swarm-controller-clients";
# Quick links to swarm services, built from `swarm.*` options alone: every
# host evaluates the same entries, whichever host runs each service. One
# entry per service; reading a `deploy.*` option here would tie a link to
# where its service runs.
swarmServiceLinks =
map
(s: {
inherit (s) label icon;
url = "https://${s.domain}/";
})
[
{
label = "Forge";
icon = "🔨";
inherit (swarmCfg.forge) domain;
}
{
label = "Authelia";
icon = "🔑";
inherit (swarmCfg.authelia) domain;
}
{
label = "Grafana";
icon = "📊";
inherit (swarmCfg.grafana) domain;
}
{
label = "Metrics";
icon = "📈";
inherit (swarmCfg.victoriametrics) domain;
}
{
label = "Logs";
icon = "📜";
inherit (swarmCfg.victorialogs) domain;
}
{
label = "Bao";
icon = "🔐";
inherit (swarmCfg.bao.ui) domain;
}
]
# A null `gatewayHost` means no vhost fronts the homeserver, so there is
# no web client to link to.
++ lib.optional (swarmCfg.matrix.gatewayHost != null) {
label = "Matrix";
icon = "💬";
url = "https://${swarmCfg.matrix.gatewayHost}/";
};
# Where the secret store is, and whether this host holds the controller's
# own leaf for it. ⚠️ The controller's pair, NOT `deploy.bao.clientCertFile`
# — that one is the hive reader's, and its policy scopes a hive's own
# secrets. The two are deliberately separate identities.
#
# Gated on the leaf, never on `deploy.bao.enable`: a controller three
# networks away from the store holds a leaf issued out of band and wants
# exactly this wiring. Same rule ./glue-controller-bao-identity.nix states
# for the paths themselves.
baoCfg = config.services.hyperhive.swarm.bao;
haveBaoIdentity =
deployCfg.swarm-controller.baoClientCertFile != null
&& deployCfg.swarm-controller.baoClientKeyFile != null;
# Creating a hive's role means writing to the store, so the CA alone is not
# enough — without an identity there is nothing to write it with, and the
# file would be handed to a daemon that cannot use it.
haveHiveClientCa = haveBaoIdentity && deployCfg.swarm-controller.hiveClientCaFile != null;
# `swarm_secret_client` reads these spellings explicitly rather than
# vaultrs's `VAULT_*` defaults — falling through to those builds a client
# with no identity and fails at the TLS handshake, naming neither. `%d` and
# not the paths themselves: see the `LoadCredential` below.
baoEnv =
lib.optionalAttrs haveBaoIdentity {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = "%d/bao-client.pem";
BAO_CLIENT_KEY = "%d/bao-client-key.pem";
}
// lib.optionalAttrs (haveBaoIdentity && deployCfg.bao.serverCaFile != null) {
# Absent means the system trust store — right for a real CA, wrong for
# the self-signed one ./glue-bao-tls.nix mints, which is why that file
# names this path rather than leaving it to a default.
BAO_CACERT = "%d/bao-ca.pem";
}
// lib.optionalAttrs haveHiveClientCa {
# Not a `BAO_` name: that family is `swarm_secret_client`'s, and
# `BAO_CACERT` above is the *server* CA. This is the authority the
# store should trust hives by, which the controller reads as a value
# to put in each hive's cert-auth role.
SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE = "%d/hive-client-ca.pem";
}
// lib.optionalAttrs haveBaoIdentity {
# Where agent leaves are issued. The store host sets that mount and
# role up from the same two options, which keeps the spellings equal.
SWARM_CONTROLLER_AGENT_PKI_MOUNT = deployCfg.bao.agentPkiMountPath;
SWARM_CONTROLLER_AGENT_PKI_ROLE = deployCfg.bao.agentPkiRoleName;
};
# Where `swarmctl agent create` posts to. Unconditional: this binary is
# only installed where the daemon runs, so the socket is always the one
# right below it — and it is the same `socketPath` the daemon binds, read
# once rather than restated.
controllerSocketEnv = {
SWARM_CONTROLLER_SOCKET = deployCfg.swarm-controller.socketPath;
};
forgeCfg = config.services.hyperhive.swarm.forge;
swarmDomain = config.services.hyperhive.swarm.domain;
matrixCfg = config.services.hyperhive.swarm.matrix;
# The optional TCP listener. `tcpCfg` is the swarm-wide half, evaluated
# identically on every host: the store's host writes its roles from it, the
# gateway's host issues its client leaf from it, and this host serves it.
# `tcpHost` is what only this host decides.
tcpCfg = cfg.tcp;
tcpHost = deployCfg.swarm-controller.tcp;
hiveTls = deployCfg.hive-controller.tls;
storeRetry = import ./lib/store-retry.nix { };
tcpOn = deployCfg.swarm-controller.enable && tcpCfg.enable;
# Which swarm `swarmctl login` logs in to when given none: this host's own.
# Read by `login` alone. `SWARMCTL_SWARM` would also move every socket verb
# onto the gateway, where each needs a login. A default, so an operator's
# own value still wins.
loginDefaultEnv = lib.optionalAttrs (swarmDomain != null) {
SWARMCTL_LOGIN_DEFAULT_SWARM = "https://${swarmDomain}";
};
# The controller's forge client speaks TLS to `https://${forgeCfg.domain}`,
# which the gateway serves with a leaf issued by the secret store's `pki`
# mount, whose root is generated inside the store at runtime, so nothing
# build-time can name it and it is not in the system store.
# `reqwest`/`rustls` resolves roots through `rustls-native-certs`, whose
# `SSL_CERT_FILE` *replaces* that store rather than adding to it, so
# without the assembled bundle every forge call fails `invalid peer
# certificate: UnknownIssuer` and agent creation dies at its first step.
#
# `hostUnit`: this consumer is a host service, not a container, so it reads
# the CA from the host path and the oneshot waits on `hive-tls-ca.service`
# itself. `enable`: `imports` is unconditional at the host's top level, so
# without it a hive with the controller off would get a bundle unit and a
# phantom `swarm-controller` service holding an `SSL_CERT_FILE`.
caTrust = import ./lib/hive-ca-trust.nix {
inherit lib;
tlsCfg = config.services.hyperhive.deploy.hive-controller.tls;
gatewayCfg = config.services.hyperhive.gateway;
};
# `swarmctl --swarm` (and `login`) reach the same gateway as the daemon,
# through the same `reqwest`/`rustls` roots, so the wrapper hands it the
# daemon's bundle. `--set`, not a default: a nix shell exports its own
# `SSL_CERT_FILE`, which would win and lack the swarm's root. The bundle
# holds the system CAs too, so public hosts still verify.
swarmctlTrustEnv = lib.optionalAttrs caTrust.useSelfSigned {
SSL_CERT_FILE = caTrust.bundlePathFor "swarm-controller";
};
# The controller's own OAuth2 client. It is NOT a hive: the per-hive
# clients the roster issues belong to hives, and the responder's client
# belongs to the responder. One identity per principal — the rule is that
# a principal's credentials all derive from the same identity, not that
# the swarm has one.
queueClientId = cfg.queueClientId;
# No secret here. The daemon reads its client secret from the store at
# start, under `baoEnv`'s identity, from the path authelia's mint
# writes in its container (`queue_identity.rs`).
#
# Every value here comes from an option rather than from what happens to
# run on this host. The queue is not optional for a controller, but
# *co-location with it* is — "controller needs queue, but it may not run on
# same host if configured properly by operator" is the rule. The
# defaults below cover the co-located case so nobody types a path they
# didn't need to; the assertions cover the split case so nobody gets a
# daemon that silently never connects.
queueEnv = {
SWARM_CONTROLLER_NATS_URL = deployCfg.swarm-controller.queue.natsUrl;
SWARM_CONTROLLER_OIDC_TOKEN_ENDPOINT = deployCfg.swarm-controller.queue.tokenEndpoint;
SWARM_CONTROLLER_OIDC_CLIENT_ID = queueClientId;
};
# Unconditional: every swarm has a forge, reached by name wherever it runs.
# The token is not here: the daemon reads it from the swarm's secret store at
# start (`forge.rs`), and re-reads it in the background until it is there.
forgeEnv = {
SWARM_CONTROLLER_FORGE_URL = "https://${forgeCfg.domain}";
# `agent-configs` org avatar for the forge-objects pass
# (`forge/objects.rs`). Only meaningful with forge access, hence here.
SWARM_CONTROLLER_CONFIG_ORG_AVATAR_PNG =
if deployCfg.swarm-controller.configOrgAvatarPng != null then
"${deployCfg.swarm-controller.configOrgAvatarPng}"
else
"${config.services.hyperhive.deploy.hive-controller.assets}/share/hyperhive/branding/agent-configs.png";
# The swarm services' own forge accounts' avatars (`forge/service_avatars.rs`).
SWARM_CONTROLLER_SERVICE_ICON_DIR = "${config.services.hyperhive.deploy.hive-controller.assets}/share/hyperhive/branding";
};
# How the forge must address this controller to deliver a swarm-wide
# webhook: `deploy.swarm-controller.publicUrl`. ./swarm-ui.nix declares the
# vhost and the `/webhook/forge/` location inside it on every host that runs
# this daemon, so the only gate left is a URL to name it by.
#
# 🔑 The gate is the point, not a detail. The daemon registers hooks only
# when this is set, and a hook whose `target_url` nothing answers is worse
# than no hook at all — forgejo keeps the registration, marks every
# delivery failed, and the hook still reads as configured.
webhookEnv = lib.optionalAttrs (deployCfg.swarm-controller.publicUrl != null) {
SWARM_CONTROLLER_PUBLIC_URL = deployCfg.swarm-controller.publicUrl;
};
# What the daemon needs to open its TCP listener (`swarm-controller/src/
# tcp_tls.rs`). All five or none: the daemon refuses a partial set. The
# client CA is the store's services root, the one the gateway's client leaf
# chains to, from the file `hive-tls-ca` already keeps on every self-signed
# gateway host.
tcpEnv = lib.optionalAttrs tcpOn {
SWARM_CONTROLLER_TCP_LISTEN = "${tcpHost.listenAddress}:${toString tcpCfg.port}";
SWARM_CONTROLLER_TCP_CERT = "${tcpHost.tlsDir}/cert.pem";
SWARM_CONTROLLER_TCP_KEY = "${tcpHost.tlsDir}/key.pem";
SWARM_CONTROLLER_TCP_CLIENT_CA = "${hiveTls.stateDir}/swarm-services-root.pem";
SWARM_CONTROLLER_TCP_CLIENT_NAME = tcpCfg.clientName;
};
# Not a secret to deliver — `swarm-authelia-bridge`'s own bearer check
# is satisfied by THIS daemon's existing queue OIDC identity
# (`queueEnv` above): "one identity per principal" already covers this,
# so there is nothing new to mint or copy, just the bridge's address.
authBridgeEnv = lib.optionalAttrs (deployCfg.swarm-controller.authBridgeUrl != null) {
SWARM_CONTROLLER_AUTH_BRIDGE_URL = deployCfg.swarm-controller.authBridgeUrl;
};
# The swarm's homeserver: where `MintAgentMatrixAccount` creates each
# agent's own account. Gated on the option resolving: a swarm with no
# domain has no homeserver URL, and the mint node then fails by name.
matrixHomeserverEnv = lib.optionalAttrs (deployCfg.swarm-controller.matrixHomeserverUrl != null) {
SWARM_CONTROLLER_MATRIX_HOMESERVER_URL = deployCfg.swarm-controller.matrixHomeserverUrl;
};
# What `internal/meta` renders into every agent's build
# (swarm-controller/src/meta_flake.rs `Settings`): the names and values
# ./hive-c0re/environment.nix forwards, minus the per-host ones. One flake
# serves every hive, so a host's own answer here would reach all of them.
# `HIVE_AGENT_NATS_URL` is per host and reaches the container through the
# hive facts instead.
#
# The store-backed values are gated on this controller's store identity,
# where a hive gates them on its own: the swarm has a store when its
# controller can reach one.
metaEnv = {
SWARM_CONTROLLER_META_FLAKE = builtins.toJSON {
vars = {
HIVE_FORGE_URL = "http://${forgeCfg.domain}";
}
// lib.optionalAttrs (matrixCfg.gatewayHost != null) {
HIVE_MATRIX_URL = "http://${matrixCfg.gatewayHost}";
}
// lib.optionalAttrs (swarmCfg.name != null) {
HYPERHIVE_SWARM_NAME = swarmCfg.name;
}
// lib.optionalAttrs haveBaoIdentity {
HIVE_AGENT_LOGS_QUERY_URL = "https://${swarmCfg.victorialogs.domain}/select/logsql/query";
HIVE_AGENT_METRICS_URL = "https://${swarmCfg.victoriametrics.domain}";
HIVE_AGENT_BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
HIVE_AGENT_BAO_SERVICES_PKI_MOUNT = deployCfg.bao.servicesPkiMountPath;
};
# Where the pinned inputs are fetched from: the builder's credential
# helper answers only `https` for the forge.
git_base = "https://${forgeCfg.domain}";
github_disabled = !swarmCfg.github.enable;
};
};
# The rev this controller runs, which `internal/meta` pins hyperhive to once
# it descends from the committed pin; null on a dirty tree. It is a file the
# daemon re-reads on every meta update, not part of the unit: the rev differs
# on every commit, and a changed unit restarts the controller and drops its
# job graph.
runningRevFile = builtins.toJSON {
hyperhive_rev = deployCfg.hive-controller.hyperhiveRev;
};
# The swarm's own display name, for `GET /api/swarm` (swarm-ui's chrome).
# Gated on the option resolving, not defaulted to an empty string: an
# operator who never named the swarm gets `name: null` from the
# endpoint, not a blank label rendered as if it meant something.
swarmNameEnv = lib.optionalAttrs (config.services.hyperhive.swarm.name != null) {
SWARM_CONTROLLER_NAME = config.services.hyperhive.swarm.name;
};
# Wrapped rather than documented: every one of these values is derived
# from an option this deployment already set, so making the operator
# re-supply them on the command line would be asking them to repeat the
# config back to it — and to get it wrong the day one of them changes.
swarmctlConfigured = pkgs.symlinkJoin {
name = "swarmctl-configured";
paths = [ deployCfg.swarm-controller.swarmctlPackage ];
nativeBuildInputs = [ pkgs.makeWrapper ];
postBuild = ''
wrapProgram $out/bin/swarmctl ${
lib.concatStringsSep " " (
lib.mapAttrsToList (name: value: "--set ${name} ${lib.escapeShellArg value}") (
controllerSocketEnv // swarmctlTrustEnv // deployCfg.swarm-controller.swarmctlEnvironment
)
++ lib.mapAttrsToList (
name: value: "--set-default ${name} ${lib.escapeShellArg value}"
) loginDefaultEnv
)
}
'';
};
in
{
imports = [
(caTrust.trustBundle {
inherit pkgs;
name = "swarm-controller";
consumers = [ "swarm-controller" ];
hostUnit = true;
enable = deployCfg.swarm-controller.enable;
})
(lib.mkRemovedOptionModule [ "services" "hyperhive" "deploy" "swarm-controller" "forgeTokenFile" ]
''
The controller reads its forge token from the swarm's secret store, under
its own certificate (deploy.swarm-controller.baoClientCertFile), and
forgejo-swarm-controller-account in the forge's container puts it
there. Remove this definition; the file it named is read by nothing now
and can be deleted.
''
)
];
options.services.hyperhive.swarm.controller = {
queueClientId = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "swarm-controller";
description = ''
The OAuth2 client id the controller presents to the swarm queue.
Read-only: it is the client the host running authelia registers,
whichever host runs the controller, published so the auth-callout
responder can be told which client may read every hive's key without
repeating the string.
The responder decides that from a client id, and a client id it
does not recognise is **denied**. A denial reaches a NATS client
as a timeout rather than an error, and a controller that cannot
read looks exactly like a swarm where no hive has reported yet —
so a drift between these two spellings is invisible at the point
it is introduced and misattributed everywhere it shows up.
'';
};
# `enable` is `services.hyperhive.deploy.swarm-controller.enable` — see
# ./deploy.nix. `services.hyperhive.deploy.singleHostSwarm` asserts
# it, and that is not an exception to "not derived from
# deploy.hive-controller.enable": that mode says "this box is the whole
# deployment", which answers the topology question outright, where
# `enable` alone never can. Both packages are in that same
# `deploy.swarm-controller` block. What is declared here is what the
# daemon IS.
staleAfterSeconds = lib.mkOption {
type = lib.types.ints.positive;
default = 120;
description = ''
How old a hive's last status snapshot may be before
`GET /api/hives/status` reports it as `stale` rather than `fresh`.
This is a statement about how often hives *publish*, not about how
patient a reader is — set it above the publishing cadence or every
hive reads stale between offers. It is an option and not a
constant precisely because that cadence is a property of the
deployment.
Freshness is derived when the endpoint is read, never stored, so
changing this takes effect for the next request; no hive has to
re-publish anything.
'';
};
# Swarm tier, not `deploy.*`: the store's host writes the two roles below,
# the host fronting the controller issues the gateway's client leaf, and the
# controller's host serves — three hosts that must agree on one set of
# names, so each reads them from the option every host evaluates alike.
tcp = {
enable = lib.mkEnableOption ''
the controller's TCP listener, TLS with a required client certificate.
Off: the controller serves its unix socket only, which is what the
gateway on the same host reaches it through.
On: the gateway's controller locations dial {option}`domain` over TLS
and present a client leaf, and the controller accepts that one
identity. The unix socket keeps working beside it. The listener
reaches only the gateway: the controller trusts its transport and
checks nobody per request, so a certificate the swarm root signed for
anything else is refused at the handshake.
Set it identically on every host of the swarm. The store's host
writes the two roles from it, the controller's host issues its server
leaf, and the host fronting the controller issues the gateway's client
leaf, each through an identity it already holds.
'';
domain = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "controller.example.org";
description = ''
The name the gateway dials the controller by. It becomes the server
certificate's name (the store's `swarm-controller-tls` role issues
for this name alone) and what nginx verifies the controller against.
It must resolve, from the gateway's host, to an address the
controller listens on. Required when {option}`enable` is set.
'';
};
port = lib.mkOption {
type = lib.types.port;
default = 9443;
description = "TCP port the controller listens on and the gateway dials.";
};
clientName = lib.mkOption {
type = lib.types.nullOr lib.types.str;
readOnly = true;
default = if swarmDomain == null then null else "gateway-client.${swarmDomain}";
defaultText = lib.literalExpression ''"gateway-client.''${services.hyperhive.swarm.domain}"'';
description = ''
The one name a client certificate must carry for the controller to
accept it. The store's `swarm-controller-gateway-client` role issues for this
name alone, with the `clientAuth` usage and not `serverAuth`, so the
swarm-services leaf the gateway serves browsers with does not
qualify.
'';
};
serverPkiRole = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "swarm-controller-tls";
description = ''
The store's `pki` role that issues the controller's server leaf.
Starts with `swarm-`: the granter that writes it may write roles
under that prefix only.
'';
};
clientPkiRole = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "swarm-controller-gateway-client";
description = ''
The store's `pki` role that issues the gateway's client leaf. Starts
with `swarm-`, as {option}`serverPkiRole` does.
'';
};
};
};
# What stays above is what the controller IS to the swarm:
# the client id it is registered under, and how stale a
# snapshot may read. What the host running it decides is here — the quick
# links its UI serves, the accounts its matrix admin sync leaves alone, which two
# builds it runs, where its socket lives, and the three credentials it reads
# off local disk. `enable` already lives in ./deploy.nix, which also carries
# the renames.
#
# ⚠️ `deploy.swarm-controller`, NOT `deploy.hive-controller`. Both exist:
# the latter is hive-c0re's namespace, where `tls.*` lives. A definition on
# the wrong one lands on a live, unrelated option rather than failing.
options.services.hyperhive.deploy.swarm-controller = {
links = lib.mkOption {
type = lib.types.listOf (
lib.types.submodule {
options = {
label = lib.mkOption {
type = lib.types.str;
description = "Display label for the link.";
};
icon = lib.mkOption {
type = lib.types.str;
default = "";
description = "Optional icon emoji or short glyph.";
};
url = lib.mkOption {
type = lib.types.str;
description = "Full URL.";
};
};
}
);
default = [ ];
example = lib.literalExpression ''
[ { label = "Wiki"; icon = "📖"; url = "https://wiki.example.com/"; } ]
'';
description = ''
Quick links to swarm-wide services, surfaced by the swarm UI's
links menu (`GET /api/links`). Same shape and same
zero-code-change-to-extend idea as `services.hyperhive.agent.dashboardLinks`
(`nix/agent-modules/dashboard-links.nix`), one level up: rather
than one fixed list, definitions merge with the list-merge idiom
`services.hyperhive.gateway.localNames` uses, so an operator can
add arbitrary extra entries here directly with no swarm-controller
or swarm-ui change.
The Forge, Authelia, Grafana, Metrics and Logs entries come from
`services.hyperhive.swarm.<service>.domain`, the Bao entry from
`services.hyperhive.swarm.bao.ui.domain`, and the Matrix entry
from `services.hyperhive.swarm.matrix.gatewayHost` when it is set,
so they are present whichever host runs each service.
`swarm-ui.nix` contributes its own API-docs entry, since the
swarm UI runs wherever the controller does.
Read only on the host that runs the controller.
'';
};
matrixAdminSync.ignoreUsers = lib.mkOption {
type = lib.types.listOf (lib.types.strMatching "@[^:[:space:]]+:[^[:space:]]+");
default = [ ];
example = [ "@mara:example.org" ];
description = ''
Full matrix user ids the controller's operator admin sync never
revokes homeserver admin from. The sync revokes admin only from
accounts it made admin itself, once their owner leaves authelia's
`admins` group; an account listed here keeps admin regardless.
'';
};
queue = {
natsUrl = lib.mkOption {
type = lib.types.str;
default = "tls://${natsCfg.domain}:${toString natsCfg.port}";
defaultText = lib.literalExpression ''"tls://''${swarm.nats.domain}:''${swarm.nats.port}"'';
example = "tls://nats.example.com:4222";
description = ''
Where the controller reaches the swarm queue.
Defaults to the queue's name, which is the same URL on every
host: the queue's own host resolves it locally, and any other
through the operator's DNS.
Empty means unset, which the assertion below refuses — a
controller with no queue is not a lighter controller.
'';
};
tokenEndpoint = lib.mkOption {
type = lib.types.str;
default = "${autheliaCfg.url}/api/oidc/token";
defaultText = lib.literalExpression ''"''${swarm.authelia.url}/api/oidc/token"'';
description = ''
The OIDC token endpoint the controller mints its own access
token from.
Derived from `swarm.authelia.url`, which is the half of the
authelia module that exists on every host — so this is already
correct for a remote provider as long as that URL is set.
'';
};
};
package = lib.mkOption {
type = lib.types.package;
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-controller";
description = ''
swarm-controller package. Wired by default from this flake's own
package set (see `flake.nix`); override to run a different build.
'';
};
swarmctlPackage = lib.mkOption {
type = lib.types.package;
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarmctl";
description = ''
`swarmctl` package — the swarm operator's CLI, installed on this
host alongside the daemon and wrapped with the paths it needs.
A separate option from `package` rather than a second binary in
the same derivation: the CLI runs as root and acts directly,
the daemon runs unprivileged and serves a socket, and pinning one
without the other is a thing an operator may legitimately want.
'';
};
socketPath = lib.mkOption {
type = lib.types.str;
default = "/run/swarm-controller/controller.sock";
readOnly = true;
description = ''
Unix socket the daemon serves on, and the path the gateway's nginx
proxies to unless
`swarm.controller.tcp.enable` points it at the TCP listener.
The **directory** is the access control here, not the socket mode:
the socket itself is `0666`, exactly as hive-c0re publishes the
per-agent sockets, and the directory is `0750` owned by the
`${clientsGroup}` group. Members of that group (the gateway's
nginx, for `/api/` and `/webhook/forge/`) and root (`swarmctl`,
which bypasses mode bits) can reach the socket; every other local
user gets `EACCES` at the directory. What keeps `0666` safe is that
the directory holds exactly one socket.
Moving this path under a directory that carries anything else —
`/run/hyperhive`, which holds the host admin socket, above all —
exposes whatever else lives there to the same reachability.
Changing this therefore means re-checking what else lives in the
new directory, not just the daemon.
'';
};
configOrgAvatarPng = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = ''
PNG uploaded once as the `agent-configs` Forgejo org's avatar by
the controller's swarm-wide forge-objects pass (one-shot,
marker-guarded — delete `forge-config-org-avatar-set` under the
controller's state directory to force a re-upload after changing
this). Defaults (`null`) to the bundled `agent-configs.png` from
{option}`services.hyperhive.deploy.hive-controller.assets`. Set this to override just
the org avatar without replacing the whole `assets` package.
'';
};
authBridgeUrl = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = "${autheliaCfg.url}/bridge";
defaultText = lib.literalExpression ''"''${services.hyperhive.swarm.authelia.url}/bridge"'';
example = "https://auth.example.com/bridge";
description = ''
Where `swarm-authelia-bridge` (the only writer of authelia's
users database) answers. The default is the `/bridge/` location
on the authelia vhost, the same address from every host whether
or not it runs `swarm-authelia`; the daemon appends `/requests`.
No new credential to configure: the bearer token presented to
the bridge is minted from THIS daemon's own existing queue OIDC
identity — its endpoints stay in `deploy.swarm-controller.queue`, its secret is
read from the swarm's secret store — "one identity per principal"
already covers it. `null` means no agent-identity support:
`CreateIdentity` jobs fail with a clear "no auth bridge
configured here" error rather than the daemon refusing to
start, the same graceful-absence shape forge access has.
'';
};
matrixHomeserverUrl = lib.mkOption {
type = lib.types.nullOr lib.types.str;
# `gatewayHost`'s own vhost, not a re-derivation of its default: a
# deployment that pins `gatewayHost` (e.g. to the per-hive name) still
# has one homeserver, and this
# option has to name the same one.
default = if matrixCfg.gatewayHost == null then null else "https://${matrixCfg.gatewayHost}";
defaultText = lib.literalExpression ''"https://''${services.hyperhive.swarm.matrix.gatewayHost}"'';
example = "https://matrix.example.org";
description = ''
Client-server API base of the swarm's homeserver. The controller
creates each agent's own matrix account there, with the swarm's
appservice token, and stores its token where the agent reads it.
`null` leaves agents with no matrix account: the mint node fails,
naming this option's variable, and the agent is deployed anyway.
Also the swarm-wide default homeserver for `PUT
.../matrix-accounts/{account}` requests that omit their own
`homeserver` — see that route's own doc comment
(`swarm-controller/src/matrix_account.rs`) for why the field is
optional in token mode and what omitting it currently resolves to.
That route does not consult it yet.
'';
};
publicUrl = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = if swarmDomain == null then null else "https://${swarmDomain}";
defaultText = lib.literalExpression ''"https://''${services.hyperhive.swarm.domain}"'';
example = "https://swarm.example.org";
description = ''
The URL the forge reaches this controller at to deliver webhooks,
which the daemon registers the swarm's hooks against
(`SWARM_CONTROLLER_PUBLIC_URL`). It names the vhost that fronts the
controller, not the controller: the apex gateway on this host answers
it whether the controller is behind its socket or its TCP listener.
Set it when another URL fronts the controller.
`null` registers no hooks.
'';
};
tcp = {
listenAddress = lib.mkOption {
type = lib.types.str;
default = "0.0.0.0";
example = "10.0.0.5";
description = ''
Address the controller binds when
{option}`services.hyperhive.swarm.controller.tcp.enable` is set. An
IPv6 address is written in brackets. Binding a port opens nothing:
{option}`openFirewall` decides who can reach it, and the TLS
handshake decides who may talk once they do.
'';
};
openFirewall = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Open {option}`services.hyperhive.swarm.controller.tcp.port` in this
host's firewall. Off by default: whether the controller is reachable
from off-host is the operator's decision, and a gateway on this same
host needs nothing opened. Turn it on when the gateway that fronts
the controller runs elsewhere.
'';
};
tlsDir = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "/var/lib/swarm-controller-tls";
description = ''
Where `swarm-controller-tls.service` writes the server leaf and key
the listener serves: `cert.pem` and `key.pem`, the key readable by
the controller's group alone. The daemon re-reads both when they
change, so a rotation needs no restart.
'';
};
gatewayClientCertFile = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "${hiveTls.stateDir}/controller-client.pem";
defaultText = lib.literalExpression ''"''${deploy.hive-controller.tls.stateDir}/controller-client.pem"'';
description = ''
The client certificate the gateway on this host presents to the
controller's listener. `swarm-controller-gateway-client.service`
issues it from the store, and the gateway's controller locations
name it in `proxy_ssl_certificate`.
'';
};
gatewayClientKeyFile = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "${hiveTls.stateDir}/controller-client-key.pem";
defaultText = lib.literalExpression ''"''${deploy.hive-controller.tls.stateDir}/controller-client-key.pem"'';
description = ''
Private key for {option}`gatewayClientCertFile`, mode `0640` and group
`nginx`, the way the gateway's own key is readable.
'';
};
};
swarmctlEnvironment = lib.mkOption {
type = lib.types.attrsOf lib.types.str;
default = { };
internal = true;
description = ''
Variables the installed `swarmctl` wrapper sets, on top of the
controller's socket path. Empty unless another module on this host
supplies some: ./glue-controller-env.nix adds the configured authelia
where authelia runs here.
'';
};
baoClientCertFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "/var/lib/swarm-controller/bao-client.pem";
description = ''
Certificate this daemon presents to the swarm's secret store. Its
subject has to match
{option}`services.hyperhive.deploy.bao.controllerCommonName`, which is
what the store's cert-auth role matches on.
Distinct from {option}`services.hyperhive.deploy.bao.clientCertFile` on
purpose, and not an oversight: that one is **this host as a reader** and
carries the hive's name, while the controller's policy lets it create
roles for every hive. One certificate serving both would hand that power
to whatever else reads the store here.
Defaulted by `glue-controller-bao-identity.nix` where this deployment
mints its own; elsewhere the leaf is issued out of band and named here.
'';
};
baoClientKeyFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "/var/lib/swarm-controller/bao-client-key.pem";
description = ''
Private key for {option}`services.hyperhive.deploy.swarm-controller.baoClientCertFile`.
Both or neither — a certificate with no key authenticates nothing.
'';
};
hiveClientCaFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "/var/lib/swarm-bao-pki/ca.pem";
description = ''
Authority whose leaves are hive client certificates, so that this
daemon can create each hive's cert-auth role. The store matches a
presented certificate against the role's copy of this, which is why the
role carries the authority itself rather than a path to it.
The same material as {option}`services.hyperhive.deploy.bao.clientCaFile`,
named separately because that option is the **store's**: it says which
readers the store trusts, on the host that runs the store. A controller
runs anywhere, so it names its own copy, and a glue module supplies this
as a `mkDefault` where the two are co-located.
Public material — a certificate authority, not a key — so unlike every
other credential here it leaks nothing. It stays a path anyway, because
the file it names is the one the store already installs.
`null` leaves hive roles uncreated, which is the state a swarm is in
before anyone has onboarded a hive.
'';
};
};
config = lib.mkIf deployCfg.swarm-controller.enable {
services.hyperhive.deploy.swarm-controller.links = swarmServiceLinks;
users.users.swarm-controller = {
isSystemUser = true;
group = "swarm-controller";
description = "hyperhive swarm-controller daemon";
};
users.groups.swarm-controller = { };
users.groups.${clientsGroup} = { };
# Through the unit, not `users.users.nginx.extraGroups`: systemd resolves
# a unit's groups at start, and only a changed unit file makes the switch
# restart nginx (it does not reload) onto the new group.
systemd.services.nginx = lib.mkIf config.services.nginx.enable {
serviceConfig.SupplementaryGroups = [ clientsGroup ];
};
# Read by `meta_flake::running_rev`.
environment.etc."hyperhive/swarm-controller.json".text = runningRevFile;
# Installed host-wide, not into the daemon's unit: `swarmctl` is run
# by a human on this box and acts as root, so the daemon's sandbox is
# exactly what it must not inherit.
environment.systemPackages = [ swarmctlConfigured ];
# A controller without a queue is not a lighter controller, it is a
# broken one: `/api/hives/status` reads the KV, so its whole aggregate
# half is off. The operator's rule is that the controller needs a queue
# but may not run on the same host as one, given it is configured
# properly — so the queue is required and the co-location is not.
#
# ⚠️ These fail at EVAL on purpose. The state they replace is worse than
# an error: with the coordinates absent the daemon started cleanly,
# served `/api/hives`, and silently never connected — nothing logged,
# because on an all-local deployment nothing was wrong.
assertions = [
{
assertion = deployCfg.swarm-controller.queue.natsUrl != "";
message = ''
services.hyperhive.deploy.swarm-controller.queue.natsUrl is unset.
It defaults to the queue's name; something in this configuration
set it to "". Remove that setting, or set the URL.
'';
}
{
assertion = deployCfg.swarm-controller.queue.tokenEndpoint != "";
message = ''
services.hyperhive.deploy.swarm-controller.queue.tokenEndpoint is unset.
It defaults from services.hyperhive.swarm.authelia.url, so this
only happens when it has been cleared by hand. The controller
mints its own access token before it may connect to the queue,
so it needs to know where the swarm's identity provider lives —
set this endpoint directly.
'';
}
{
assertion = haveBaoIdentity;
message = ''
services.hyperhive.deploy.swarm-controller.baoClientCertFile and
baoClientKeyFile must both be set.
The controller reads its queue client secret from the swarm's
secret store, logging in with this certificate. They default to
the leaf the store mints when it runs on this host; elsewhere,
issue a leaf whose CN is
services.hyperhive.deploy.bao.controllerCommonName and name it
in both options.
'';
}
{
assertion = !tcpCfg.enable || (tcpCfg.domain != null && tcpCfg.clientName != null);
message = ''
services.hyperhive.swarm.controller.tcp.enable needs
services.hyperhive.swarm.controller.tcp.domain, the name the gateway
dials the controller by, and services.hyperhive.swarm.domain, which
the gateway's client identity is named under. Set both, or leave the
TCP listener off.
'';
}
{
assertion = !tcpCfg.enable || caTrust.useSelfSigned;
message = ''
services.hyperhive.swarm.controller.tcp.enable needs the self-signed
gateway mode: the controller trusts the secret store's services root
for client certificates, and only a self-signed gateway host keeps
that root on disk. Leave the TCP listener off where the gateway
serves an operator certificate or ACME.
'';
}
];
systemd.services.swarm-controller = {
description = "hyperhive swarm-level controller daemon";
wantedBy = [ "multi-user.target" ];
# The listener's certificate files must exist when the daemon opens it. A
# start that finds them missing fails and `Restart=` tries again.
after = [ "network.target" ] ++ lib.optional tcpOn "swarm-controller-tls.service";
wants = lib.optional tcpOn "swarm-controller-tls.service";
serviceConfig = {
ExecStart = "${deployCfg.swarm-controller.package}/bin/swarm-controller";
# The store identity, same shape and same reason as hive-c0re's: the
# key is root-owned `0600` and this daemon runs as `swarm-controller`,
# so it never gets read access to the original file.
LoadCredential =
lib.optionals haveBaoIdentity [
"bao-client.pem:${deployCfg.swarm-controller.baoClientCertFile}"
"bao-client-key.pem:${deployCfg.swarm-controller.baoClientKeyFile}"
]
++ lib.optional (
haveBaoIdentity && deployCfg.bao.serverCaFile != null
) "bao-ca.pem:${deployCfg.bao.serverCaFile}"
++ lib.optional haveHiveClientCa "hive-client-ca.pem:${deployCfg.swarm-controller.hiveClientCaFile}";
User = "swarm-controller";
Group = "swarm-controller";
# Also what retries a start that gave up waiting for the secret store
# (`queue_identity.rs`).
Restart = "on-failure";
RestartSec = "5s";
# `/run/swarm-controller` — its own directory, holding only the
# socket. See `socketPath`'s description for why that is a security
# property and not tidiness.
RuntimeDirectory = "swarm-controller";
# 0750 with the group set to `clientsGroup` below: only nginx and
# root can reach the socket inside.
RuntimeDirectoryMode = "0750";
# systemd re-chowns the runtime directory to `User:Group` while
# setting up every command, `ExecStart` included, so a chgrp in
# `ExecStartPre` is undone. The daemon is not a member of
# `clientsGroup` and cannot chgrp it itself; `+` runs this as root,
# after the last chown.
ExecStartPost = "+${pkgs.coreutils}/bin/chgrp ${clientsGroup} ${builtins.dirOf deployCfg.swarm-controller.socketPath}";
# Preserved across restarts so the path never vanishes from under a
# running nginx. The daemon unlinks a stale socket on start, which
# is what makes preservation safe.
RuntimeDirectoryPreserve = "yes";
StateDirectory = "swarm-controller";
StateDirectoryMode = "0750";
# Nothing here needs a writable filesystem, real privileges, or a
# view of the rest of the machine.
PrivateTmp = true;
ProtectSystem = "strict";
ProtectHome = true;
NoNewPrivileges = true;
PrivateDevices = true;
ProtectKernelTunables = true;
ProtectKernelModules = true;
ProtectControlGroups = true;
# `AF_UNIX` for the socket this daemon serves on, plus what its
# outbound clients need: it mints authelia tokens and calls the forge
# over HTTPS (`auth.rs`, `forge.rs`) and reaches the queue over NATS
# (`startup.rs`, `status.rs`). `AF_NETLINK` because glibc's
# `getaddrinfo` opens a netlink socket to enumerate local addresses
# before it will return one.
#
# ⚠️ This list is a CLAIM ABOUT WHAT THE DAEMON DOES, so it goes stale
# the moment the daemon grows a client — and it goes stale in the
# worst available way: a blocked family makes `socket()` return
# EAFNOSUPPORT, i.e. "Address family not supported by protocol", so
# the error names the protocol and never the sandbox that refused it.
# This was `AF_UNIX`-only while the daemon merely served its socket;
# all three clients above arrived later, and the restriction was not
# revisited. Add the family when you add the client.
RestrictAddressFamilies = [
"AF_UNIX"
"AF_INET"
"AF_INET6"
"AF_NETLINK"
];
};
# Queue coordinates (`queueEnv`), forge coordinates (`forgeEnv`), and
# the auth-bridge address (`authBridgeEnv`) merge in last. The
# daemon refuses a PARTIAL set of any of them rather than treating
# it as absent, which is why each is built as one attrset and never
# assigned individually.
#
# They differ in how absence is prevented: the queue's is checked by
# the assertions above, because a controller without a queue is
# broken rather than lighter; the auth bridge's is genuinely optional
# and stays gated on its own option resolving. Forge's is always set.
environment = {
SWARM_CONTROLLER_SOCKET = deployCfg.swarm-controller.socketPath;
# The swarm's hive directory, JSON-encoded — the full directory
# (this daemon has no "self" hive to exclude) rather than
# peers-minus-self. Consumed by `GET /api/hives`
# (swarm-controller/src/startup.rs::load_hives).
# The one blacklist of names an agent may not take, shared verbatim
# with hive-c0re and with the collector's owner assertion — see
# `nix/reserved-names.nix`. Whitespace-separated rather than JSON
# like its neighbour below, because every entry is an `Ident` and so
# cannot contain a space; the neighbour carries objects and has no
# such option.
HIVE_RESERVED_NAMES = lib.concatStringsSep " " (import ../reserved-names.nix);
SWARM_CONTROLLER_HIVES = builtins.toJSON (
lib.mapAttrsToList (name: h: {
inherit name;
inherit (h) domain;
dashboard = config.services.hyperhive.swarm.dashboardDomains.${name};
}) config.services.hyperhive.swarm.hives
);
# The merged links list — see `links`' description above for who
# contributes to it. Consumed by `GET /api/links`
# (swarm-controller/src/startup.rs::load_links).
SWARM_CONTROLLER_LINKS = builtins.toJSON deployCfg.swarm-controller.links;
# Staleness threshold for `GET /api/hives/status` — see
# `staleAfterSeconds`' description. Set unconditionally rather
# than inside `queueEnv`: it is not a queue coordinate, and
# nothing about it is unsafe to define on a host whose queue is
# off (the daemon just has nothing to apply it to).
SWARM_CONTROLLER_STALE_AFTER_SECS = toString cfg.staleAfterSeconds;
# The accounts the operator admin sync never revokes. Whitespace-
# separated: a matrix user id cannot contain a space. Consumed by
# swarm-controller/src/matrix_account/operator_admin.rs.
SWARM_CONTROLLER_MATRIX_ADMIN_SYNC_IGNORE_USERS = lib.concatStringsSep " " deployCfg.swarm-controller.matrixAdminSync.ignoreUsers;
# Where `GET /api/alerts` reads alerts (swarm-controller/src/alerts.rs).
# Every swarm has one Grafana, reached by name wherever it runs. The
# token is in the store, written by `swarm-bao-grafana-controller-token`
# on Grafana's host.
SWARM_CONTROLLER_GRAFANA_URL = "https://${swarmCfg.grafana.domain}";
# The names squid allows every authenticated agent by default. The
# allow-list route refuses them as entries
# (swarm-controller/src/egress_allow.rs). A hive that overrides
# `deploy.hive-controller.squid.baseAllowedDomains` is not reflected
# here; its hive-c0re skips the entries its own list covers.
SWARM_CONTROLLER_SQUID_BASE_DOMAINS = lib.concatStringsSep " " (
import ./lib/squid-base-domains.nix
);
}
// queueEnv
// forgeEnv
// webhookEnv
// authBridgeEnv
// swarmNameEnv
// matrixHomeserverEnv
// baoEnv
// metaEnv
// tcpEnv;
};
networking.firewall.allowedTCPPorts = lib.mkIf (tcpOn && tcpHost.openFirewall) [ tcpCfg.port ];
# The listener's server leaf, from the store's `${tcpCfg.serverPkiRole}`
# role under the controller's own store identity — the one it already logs
# in with, whose policy gains this one issue path when the listener is on.
# Same shape as the queue's leaf unit (./swarm-nats.nix,
# `swarm-bao-nats-tls`): the daemon re-reads the files, so rotating needs
# no restart, which for this daemon would drop its job graph.
systemd.services.swarm-controller-tls = lib.mkIf tcpOn {
description = "Issue the swarm-controller's TCP listener certificate from the secret store's PKI";
wantedBy = [ "multi-user.target" ];
after = [
"container@${baoCfg.machine}.service"
"swarm-bao-pki.service"
"swarm-bao-controller-policy.service"
];
path = [
deployCfg.bao.package
pkgs.jq
pkgs.openssl
pkgs.coreutils
pkgs.gnugrep
];
# The store is sealed or unreachable for as long as it takes an operator
# to unseal it; see ./lib/store-retry.nix.
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
UMask = "0077";
SyslogIdentifier = "swarm-controller-tls";
TimeoutStartSec = "5min";
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = deployCfg.swarm-controller.baoClientCertFile;
BAO_CLIENT_KEY = deployCfg.swarm-controller.baoClientKeyFile;
}
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
BAO_CACERT = deployCfg.bao.serverCaFile;
};
script = ''
set -euo pipefail
d=${lib.escapeShellArg tcpHost.tlsDir}
name=${lib.escapeShellArg tcpCfg.domain}
install -d -m 0750 -g swarm-controller "$d"
# Missing, past half its window, or naming something other than the
# configured domain.
reissue=0
{ [ -s "$d/cert.pem" ] && [ -s "$d/key.pem" ]; } || reissue=1
openssl x509 -in "$d/cert.pem" -noout -checkend ${
toString (deployCfg.bao.servicesPkiLeafTtlHours * 3600 / 2)
} >/dev/null 2>&1 || reissue=1
openssl x509 -in "$d/cert.pem" -noout -checkhost "$name" 2>/dev/null | grep -q ' does match ' || reissue=1
if [ "$reissue" = 0 ]; then
echo "controller listener leaf valid for $name — leaving it alone"
exit 0
fi
err="$(mktemp)"
resp="$(mktemp "$d/issue.json.XXXXXX")"
trap 'rm -f "$err" "$resp"' EXIT
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to the swarm secret store with the controller's certificate." >&2
cat "$err" >&2
exit 1
fi
export BAO_TOKEN
echo "requesting the controller's listener leaf for $name from the store"
if ! bao write -format=json \
${lib.escapeShellArg "${deployCfg.bao.servicesPkiMountPath}/issue/${tcpCfg.serverPkiRole}"} \
common_name="$name" > "$resp" 2>"$err"; then
echo "the store refused to issue the controller's listener certificate." >&2
cat "$err" >&2
exit 1
fi
jq -r '.data.private_key // empty' < "$resp" > "$d/key.pem.new"
jq -r '.data.certificate // empty' < "$resp" > "$d/cert.pem.new"
for f in "$d/key.pem.new" "$d/cert.pem.new"; do
if [ ! -s "$f" ]; then
echo "the store's response was missing a field: $f is empty" >&2
exit 1
fi
done
chgrp swarm-controller "$d/key.pem.new" "$d/cert.pem.new"
chmod 0640 "$d/key.pem.new" "$d/cert.pem.new"
# Key first: the daemon pairs whatever it reads, and a new key beside
# the old certificate is a pair it refuses and retries.
mv -f "$d/key.pem.new" "$d/key.pem"
mv -f "$d/cert.pem.new" "$d/cert.pem"
'';
};
systemd.timers.swarm-controller-tls = lib.mkIf tcpOn {
description = "Daily renewal check for the swarm-controller's TCP listener certificate";
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = "daily";
Persistent = true;
};
};
};
}