Watch
0
0
Fork
You've already forked hyperhive
0

gateway reaches the controller over tls with its own client leaf when the tcp listener is on

This commit is contained in:
atlas 2026-11-15 16:10:59 +01:00 • committed by mara
commit 187ed82de0

View file

@ -7,6 +7,13 @@
# nginx is the only intended client, it runs on this same host and so
# reaches the socket by path, and a socket that is never bound to an
# address cannot be reached from off-host by mistake.
#
# `swarm.controller.tcp.enable` adds a TLS listener beside the socket, which the
# gateway then dials instead of the socket. The controller checks nobody per
# request, so the listener completes a handshake for one identity only: a
# client certificate the swarm's store issued for the gateway, with the
# `clientAuth` usage. ./swarm-ui.nix still declares the vhost on the host that
# runs this daemon, so the gateway stays on that host.
{
pkgs,
lib,
@ -137,6 +144,16 @@ let
swarmDomain = config.services.hyperhive.swarm.domain;
matrixCfg = config.services.hyperhive.swarm.matrix;
# The optional TCP listener. `tcpCfg` is the swarm-wide half, evaluated
# identically on every host: the store's host writes its roles from it, the
# gateway's host issues its client leaf from it, and this host serves it.
# `tcpHost` is what only this host decides.
tcpCfg = cfg.tcp;
tcpHost = deployCfg.swarm-controller.tcp;
hiveTls = deployCfg.hive-controller.tls;
storeRetry = import ./lib/store-retry.nix { };
tcpOn = deployCfg.swarm-controller.enable && tcpCfg.enable;
# Which swarm `swarmctl login` logs in to when given none: this host's own.
# Read by `login` alone. `SWARMCTL_SWARM` would also move every socket verb
# onto the gateway, where each needs a login. A default, so an operator's
@ -215,28 +232,29 @@ let
};
# How the forge must address this controller to deliver a swarm-wide
# webhook. ./swarm-ui.nix declares the vhost and the `/webhook/forge/`
# location inside it on every host that runs this daemon, so the only gate
# left is a swarm domain to name it by.
# webhook: `deploy.swarm-controller.publicUrl`. ./swarm-ui.nix declares the
# vhost and the `/webhook/forge/` location inside it on every host that runs
# this daemon, so the only gate left is a URL to name it by.
#
# 🔑 The gate is the point, not a detail. The daemon registers hooks only
# when this is set, and a hook whose `target_url` nothing answers is worse
# than no hook at all — forgejo keeps the registration, marks every
# delivery failed, and the hook still reads as configured.
#
# ⚠️ Correct ONLY while the vhost and this daemon share a host, and they do
# by construction: the UI's `/api/` location proxies
# `http://unix:<socketPath>`, a path that resolves nowhere else, and this
# daemon binds no TCP address at all (see the header).
#
# ⇒ The day this daemon grows a TCP listener so another host can front it,
# THIS LINE SILENTLY STOPS BEING RIGHT: hooks get registered against a vhost
# that may no longer be the one fronting the daemon, and nothing errors. That is
# the moment to add an explicit `publicUrl` option — not before, because
# until then there is exactly one derivable answer and an option would only
# be a second place to get it wrong.
webhookEnv = lib.optionalAttrs (swarmDomain != null) {
SWARM_CONTROLLER_PUBLIC_URL = "https://${swarmDomain}";
webhookEnv = lib.optionalAttrs (deployCfg.swarm-controller.publicUrl != null) {
SWARM_CONTROLLER_PUBLIC_URL = deployCfg.swarm-controller.publicUrl;
};
# What the daemon needs to open its TCP listener (`swarm-controller/src/
# tcp_tls.rs`). All five or none: the daemon refuses a partial set. The
# client CA is the store's services root, the one the gateway's client leaf
# chains to, from the file `hive-tls-ca` already keeps on every self-signed
# gateway host.
tcpEnv = lib.optionalAttrs tcpOn {
SWARM_CONTROLLER_TCP_LISTEN = "${tcpHost.listenAddress}:${toString tcpCfg.port}";
SWARM_CONTROLLER_TCP_CERT = "${tcpHost.tlsDir}/cert.pem";
SWARM_CONTROLLER_TCP_KEY = "${tcpHost.tlsDir}/key.pem";
SWARM_CONTROLLER_TCP_CLIENT_CA = "${hiveTls.stateDir}/swarm-services-root.pem";
SWARM_CONTROLLER_TCP_CLIENT_NAME = tcpCfg.clientName;
};
# Not a secret to deliver — `swarm-authelia-bridge`'s own bearer check
@ -396,6 +414,84 @@ in
'';
};
# Swarm tier, not `deploy.*`: the store's host writes the two roles below,
# the host fronting the controller issues the gateway's client leaf, and the
# controller's host serves — three hosts that must agree on one set of
# names, so each reads them from the option every host evaluates alike.
tcp = {
enable = lib.mkEnableOption ''
the controller's TCP listener, TLS with a required client certificate.
Off: the controller serves its unix socket only, which is what the
gateway on the same host reaches it through.
On: the gateway's controller locations dial {option}`domain` over TLS
and present a client leaf, and the controller accepts that one
identity. The unix socket keeps working beside it. The listener
reaches only the gateway: the controller trusts its transport and
checks nobody per request, so a certificate the swarm root signed for
anything else is refused at the handshake.
Set it identically on every host of the swarm. The store's host
writes the two roles from it, the controller's host issues its server
leaf, and the host fronting the controller issues the gateway's client
leaf, each through an identity it already holds.
'';
domain = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "controller.example.org";
description = ''
The name the gateway dials the controller by. It becomes the server
certificate's name (the store's `swarm-controller-tls` role issues
for this name alone) and what nginx verifies the controller against.
It must resolve, from the gateway's host, to an address the
controller listens on. Required when {option}`enable` is set.
'';
};
port = lib.mkOption {
type = lib.types.port;
default = 9443;
description = "TCP port the controller listens on and the gateway dials.";
};
clientName = lib.mkOption {
type = lib.types.nullOr lib.types.str;
readOnly = true;
default = if swarmDomain == null then null else "gateway-client.${swarmDomain}";
defaultText = lib.literalExpression ''"gateway-client.''${services.hyperhive.swarm.domain}"'';
description = ''
The one name a client certificate must carry for the controller to
accept it. The store's `swarm-controller-gateway-client` role issues for this
name alone, with the `clientAuth` usage and not `serverAuth`, so the
swarm-services leaf the gateway serves browsers with does not
qualify.
'';
};
serverPkiRole = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "swarm-controller-tls";
description = ''
The store's `pki` role that issues the controller's server leaf.
Starts with `swarm-`: the granter that writes it may write roles
under that prefix only.
'';
};
clientPkiRole = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "swarm-controller-gateway-client";
description = ''
The store's `pki` role that issues the gateway's client leaf. Starts
with `swarm-`, as {option}`serverPkiRole` does.
'';
};
};
};
# What stays above is what the controller IS to the swarm:
@ -610,6 +706,86 @@ in
'';
};
publicUrl = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = if swarmDomain == null then null else "https://${swarmDomain}";
defaultText = lib.literalExpression ''"https://''${services.hyperhive.swarm.domain}"'';
example = "https://swarm.example.org";
description = ''
The URL the forge reaches this controller at to deliver webhooks,
which the daemon registers the swarm's hooks against
(`SWARM_CONTROLLER_PUBLIC_URL`). It names the vhost that fronts the
controller, not the controller: the apex gateway on this host answers
it whether the controller is behind its socket or its TCP listener.
Set it when another URL fronts the controller.
`null` registers no hooks.
'';
};
tcp = {
listenAddress = lib.mkOption {
type = lib.types.str;
default = "0.0.0.0";
example = "10.0.0.5";
description = ''
Address the controller binds when
{option}`services.hyperhive.swarm.controller.tcp.enable` is set. An
IPv6 address is written in brackets. Binding a port opens nothing:
{option}`openFirewall` decides who can reach it, and the TLS
handshake decides who may talk once they do.
'';
};
openFirewall = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Open {option}`services.hyperhive.swarm.controller.tcp.port` in this
host's firewall. Off by default: whether the controller is reachable
from off-host is the operator's decision, and a gateway on this same
host needs nothing opened. Turn it on when the gateway that fronts
the controller runs elsewhere.
'';
};
tlsDir = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "/var/lib/swarm-controller-tls";
description = ''
Where `swarm-controller-tls.service` writes the server leaf and key
the listener serves: `cert.pem` and `key.pem`, the key readable by
the controller's group alone. The daemon re-reads both when they
change, so a rotation needs no restart.
'';
};
gatewayClientCertFile = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "${hiveTls.stateDir}/controller-client.pem";
defaultText = lib.literalExpression ''"''${deploy.hive-controller.tls.stateDir}/controller-client.pem"'';
description = ''
The client certificate the gateway on this host presents to the
controller's listener. `swarm-controller-gateway-client.service`
issues it from the store, and the gateway's controller locations
name it in `proxy_ssl_certificate`.
'';
};
gatewayClientKeyFile = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "${hiveTls.stateDir}/controller-client-key.pem";
defaultText = lib.literalExpression ''"''${deploy.hive-controller.tls.stateDir}/controller-client-key.pem"'';
description = ''
Private key for {option}`gatewayClientCertFile`, mode `0640` and group
`nginx`, the way the gateway's own key is readable.
'';
};
};
swarmctlEnvironment = lib.mkOption {
type = lib.types.attrsOf lib.types.str;
default = { };
@ -751,12 +927,35 @@ in
in both options.
'';
}
{
assertion = !tcpCfg.enable || (tcpCfg.domain != null && tcpCfg.clientName != null);
message = ''
services.hyperhive.swarm.controller.tcp.enable needs
services.hyperhive.swarm.controller.tcp.domain, the name the gateway
dials the controller by, and services.hyperhive.swarm.domain, which
the gateway's client identity is named under. Set both, or leave the
TCP listener off.
'';
}
{
assertion = !tcpCfg.enable || caTrust.useSelfSigned;
message = ''
services.hyperhive.swarm.controller.tcp.enable needs the self-signed
gateway mode: the controller trusts the secret store's services root
for client certificates, and only a self-signed gateway host keeps
that root on disk. Leave the TCP listener off where the gateway
serves an operator certificate or ACME.
'';
}
];
systemd.services.swarm-controller = {
description = "hyperhive swarm-level controller daemon";
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
# The listener's certificate files must exist when the daemon opens it. A
# start that finds them missing fails and `Restart=` tries again.
after = [ "network.target" ] ++ lib.optional tcpOn "swarm-controller-tls.service";
wants = lib.optional tcpOn "swarm-controller-tls.service";
serviceConfig = {
ExecStart = "${deployCfg.swarm-controller.package}/bin/swarm-controller";
@ -901,7 +1100,113 @@ in
// swarmNameEnv
// matrixHomeserverEnv
// baoEnv
// metaEnv;
// metaEnv
// tcpEnv;
};
networking.firewall.allowedTCPPorts = lib.mkIf (tcpOn && tcpHost.openFirewall) [ tcpCfg.port ];
# The listener's server leaf, from the store's `${tcpCfg.serverPkiRole}`
# role under the controller's own store identity — the one it already logs
# in with, whose policy gains this one issue path when the listener is on.
# Same shape as the queue's leaf unit (./swarm-nats.nix,
# `swarm-bao-nats-tls`): the daemon re-reads the files, so rotating needs
# no restart, which for this daemon would drop its job graph.
systemd.services.swarm-controller-tls = lib.mkIf tcpOn {
description = "Issue the swarm-controller's TCP listener certificate from the secret store's PKI";
wantedBy = [ "multi-user.target" ];
after = [
"container@${baoCfg.machine}.service"
"swarm-bao-pki.service"
"swarm-bao-controller-policy.service"
];
path = [
deployCfg.bao.package
pkgs.jq
pkgs.openssl
pkgs.coreutils
pkgs.gnugrep
];
# The store is sealed or unreachable for as long as it takes an operator
# to unseal it; see ./lib/store-retry.nix.
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
UMask = "0077";
SyslogIdentifier = "swarm-controller-tls";
TimeoutStartSec = "5min";
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = deployCfg.swarm-controller.baoClientCertFile;
BAO_CLIENT_KEY = deployCfg.swarm-controller.baoClientKeyFile;
}
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
BAO_CACERT = deployCfg.bao.serverCaFile;
};
script = ''
set -euo pipefail
d=${lib.escapeShellArg tcpHost.tlsDir}
name=${lib.escapeShellArg tcpCfg.domain}
install -d -m 0750 -g swarm-controller "$d"
# Missing, past half its window, or naming something other than the
# configured domain.
reissue=0
{ [ -s "$d/cert.pem" ] && [ -s "$d/key.pem" ]; } || reissue=1
openssl x509 -in "$d/cert.pem" -noout -checkend ${
toString (deployCfg.bao.servicesPkiLeafTtlHours * 3600 / 2)
} >/dev/null 2>&1 || reissue=1
openssl x509 -in "$d/cert.pem" -noout -checkhost "$name" 2>/dev/null | grep -q ' does match ' || reissue=1
if [ "$reissue" = 0 ]; then
echo "controller listener leaf valid for $name — leaving it alone"
exit 0
fi
err="$(mktemp)"
resp="$(mktemp "$d/issue.json.XXXXXX")"
trap 'rm -f "$err" "$resp"' EXIT
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to the swarm secret store with the controller's certificate." >&2
cat "$err" >&2
exit 1
fi
export BAO_TOKEN
echo "requesting the controller's listener leaf for $name from the store"
if ! bao write -format=json \
${lib.escapeShellArg "${deployCfg.bao.servicesPkiMountPath}/issue/${tcpCfg.serverPkiRole}"} \
common_name="$name" > "$resp" 2>"$err"; then
echo "the store refused to issue the controller's listener certificate." >&2
cat "$err" >&2
exit 1
fi
jq -r '.data.private_key // empty' < "$resp" > "$d/key.pem.new"
jq -r '.data.certificate // empty' < "$resp" > "$d/cert.pem.new"
for f in "$d/key.pem.new" "$d/cert.pem.new"; do
if [ ! -s "$f" ]; then
echo "the store's response was missing a field: $f is empty" >&2
exit 1
fi
done
chgrp swarm-controller "$d/key.pem.new" "$d/cert.pem.new"
chmod 0640 "$d/key.pem.new" "$d/cert.pem.new"
# Key first: the daemon pairs whatever it reads, and a new key beside
# the old certificate is a pair it refuses and retries.
mv -f "$d/key.pem.new" "$d/key.pem"
mv -f "$d/cert.pem.new" "$d/cert.pem"
'';
};
systemd.timers.swarm-controller-tls = lib.mkIf tcpOn {
description = "Daily renewal check for the swarm-controller's TCP listener certificate";
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = "daily";
Persistent = true;
};
};
};
}