gateway reaches the controller over tls with its own client leaf when the tcp listener is on
This commit is contained in:
parent
17fffaae79
commit
187ed82de0
1 changed files with 768 additions and 49 deletions
|
|
@ -7,6 +7,13 @@
|
|||
# nginx is the only intended client, it runs on this same host and so
|
||||
# reaches the socket by path, and a socket that is never bound to an
|
||||
# address cannot be reached from off-host by mistake.
|
||||
#
|
||||
# `swarm.controller.tcp.enable` adds a TLS listener beside the socket, which the
|
||||
# gateway then dials instead of the socket. The controller checks nobody per
|
||||
# request, so the listener completes a handshake for one identity only: a
|
||||
# client certificate the swarm's store issued for the gateway, with the
|
||||
# `clientAuth` usage. ./swarm-ui.nix still declares the vhost on the host that
|
||||
# runs this daemon, so the gateway stays on that host.
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
|
|
@ -137,6 +144,16 @@ let
|
|||
swarmDomain = config.services.hyperhive.swarm.domain;
|
||||
matrixCfg = config.services.hyperhive.swarm.matrix;
|
||||
|
||||
# The optional TCP listener. `tcpCfg` is the swarm-wide half, evaluated
|
||||
# identically on every host: the store's host writes its roles from it, the
|
||||
# gateway's host issues its client leaf from it, and this host serves it.
|
||||
# `tcpHost` is what only this host decides.
|
||||
tcpCfg = cfg.tcp;
|
||||
tcpHost = deployCfg.swarm-controller.tcp;
|
||||
hiveTls = deployCfg.hive-controller.tls;
|
||||
storeRetry = import ./lib/store-retry.nix { };
|
||||
tcpOn = deployCfg.swarm-controller.enable && tcpCfg.enable;
|
||||
|
||||
# Which swarm `swarmctl login` logs in to when given none: this host's own.
|
||||
# Read by `login` alone. `SWARMCTL_SWARM` would also move every socket verb
|
||||
# onto the gateway, where each needs a login. A default, so an operator's
|
||||
|
|
@ -215,28 +232,29 @@ let
|
|||
};
|
||||
|
||||
# How the forge must address this controller to deliver a swarm-wide
|
||||
# webhook. ./swarm-ui.nix declares the vhost and the `/webhook/forge/`
|
||||
# location inside it on every host that runs this daemon, so the only gate
|
||||
# left is a swarm domain to name it by.
|
||||
# webhook: `deploy.swarm-controller.publicUrl`. ./swarm-ui.nix declares the
|
||||
# vhost and the `/webhook/forge/` location inside it on every host that runs
|
||||
# this daemon, so the only gate left is a URL to name it by.
|
||||
#
|
||||
# 🔑 The gate is the point, not a detail. The daemon registers hooks only
|
||||
# when this is set, and a hook whose `target_url` nothing answers is worse
|
||||
# than no hook at all — forgejo keeps the registration, marks every
|
||||
# delivery failed, and the hook still reads as configured.
|
||||
#
|
||||
# ⚠️ Correct ONLY while the vhost and this daemon share a host, and they do
|
||||
# by construction: the UI's `/api/` location proxies
|
||||
# `http://unix:<socketPath>`, a path that resolves nowhere else, and this
|
||||
# daemon binds no TCP address at all (see the header).
|
||||
#
|
||||
# ⇒ The day this daemon grows a TCP listener so another host can front it,
|
||||
# THIS LINE SILENTLY STOPS BEING RIGHT: hooks get registered against a vhost
|
||||
# that may no longer be the one fronting the daemon, and nothing errors. That is
|
||||
# the moment to add an explicit `publicUrl` option — not before, because
|
||||
# until then there is exactly one derivable answer and an option would only
|
||||
# be a second place to get it wrong.
|
||||
webhookEnv = lib.optionalAttrs (swarmDomain != null) {
|
||||
SWARM_CONTROLLER_PUBLIC_URL = "https://${swarmDomain}";
|
||||
webhookEnv = lib.optionalAttrs (deployCfg.swarm-controller.publicUrl != null) {
|
||||
SWARM_CONTROLLER_PUBLIC_URL = deployCfg.swarm-controller.publicUrl;
|
||||
};
|
||||
|
||||
# What the daemon needs to open its TCP listener (`swarm-controller/src/
|
||||
# tcp_tls.rs`). All five or none: the daemon refuses a partial set. The
|
||||
# client CA is the store's services root, the one the gateway's client leaf
|
||||
# chains to, from the file `hive-tls-ca` already keeps on every self-signed
|
||||
# gateway host.
|
||||
tcpEnv = lib.optionalAttrs tcpOn {
|
||||
SWARM_CONTROLLER_TCP_LISTEN = "${tcpHost.listenAddress}:${toString tcpCfg.port}";
|
||||
SWARM_CONTROLLER_TCP_CERT = "${tcpHost.tlsDir}/cert.pem";
|
||||
SWARM_CONTROLLER_TCP_KEY = "${tcpHost.tlsDir}/key.pem";
|
||||
SWARM_CONTROLLER_TCP_CLIENT_CA = "${hiveTls.stateDir}/swarm-services-root.pem";
|
||||
SWARM_CONTROLLER_TCP_CLIENT_NAME = tcpCfg.clientName;
|
||||
};
|
||||
|
||||
# Not a secret to deliver — `swarm-authelia-bridge`'s own bearer check
|
||||
|
|
@ -396,6 +414,84 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
# Swarm tier, not `deploy.*`: the store's host writes the two roles below,
|
||||
# the host fronting the controller issues the gateway's client leaf, and the
|
||||
# controller's host serves — three hosts that must agree on one set of
|
||||
# names, so each reads them from the option every host evaluates alike.
|
||||
tcp = {
|
||||
enable = lib.mkEnableOption ''
|
||||
the controller's TCP listener, TLS with a required client certificate.
|
||||
Off: the controller serves its unix socket only, which is what the
|
||||
gateway on the same host reaches it through.
|
||||
|
||||
On: the gateway's controller locations dial {option}`domain` over TLS
|
||||
and present a client leaf, and the controller accepts that one
|
||||
identity. The unix socket keeps working beside it. The listener
|
||||
reaches only the gateway: the controller trusts its transport and
|
||||
checks nobody per request, so a certificate the swarm root signed for
|
||||
anything else is refused at the handshake.
|
||||
|
||||
Set it identically on every host of the swarm. The store's host
|
||||
writes the two roles from it, the controller's host issues its server
|
||||
leaf, and the host fronting the controller issues the gateway's client
|
||||
leaf, each through an identity it already holds.
|
||||
'';
|
||||
|
||||
domain = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "controller.example.org";
|
||||
description = ''
|
||||
The name the gateway dials the controller by. It becomes the server
|
||||
certificate's name (the store's `swarm-controller-tls` role issues
|
||||
for this name alone) and what nginx verifies the controller against.
|
||||
It must resolve, from the gateway's host, to an address the
|
||||
controller listens on. Required when {option}`enable` is set.
|
||||
'';
|
||||
};
|
||||
|
||||
port = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 9443;
|
||||
description = "TCP port the controller listens on and the gateway dials.";
|
||||
};
|
||||
|
||||
clientName = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
readOnly = true;
|
||||
default = if swarmDomain == null then null else "gateway-client.${swarmDomain}";
|
||||
defaultText = lib.literalExpression ''"gateway-client.''${services.hyperhive.swarm.domain}"'';
|
||||
description = ''
|
||||
The one name a client certificate must carry for the controller to
|
||||
accept it. The store's `swarm-controller-gateway-client` role issues for this
|
||||
name alone, with the `clientAuth` usage and not `serverAuth`, so the
|
||||
swarm-services leaf the gateway serves browsers with does not
|
||||
qualify.
|
||||
'';
|
||||
};
|
||||
|
||||
serverPkiRole = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
readOnly = true;
|
||||
default = "swarm-controller-tls";
|
||||
description = ''
|
||||
The store's `pki` role that issues the controller's server leaf.
|
||||
Starts with `swarm-`: the granter that writes it may write roles
|
||||
under that prefix only.
|
||||
'';
|
||||
};
|
||||
|
||||
clientPkiRole = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
readOnly = true;
|
||||
default = "swarm-controller-gateway-client";
|
||||
description = ''
|
||||
The store's `pki` role that issues the gateway's client leaf. Starts
|
||||
with `swarm-`, as {option}`serverPkiRole` does.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
};
|
||||
|
||||
# What stays above is what the controller IS to the swarm:
|
||||
|
|
@ -610,6 +706,86 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
publicUrl = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = if swarmDomain == null then null else "https://${swarmDomain}";
|
||||
defaultText = lib.literalExpression ''"https://''${services.hyperhive.swarm.domain}"'';
|
||||
example = "https://swarm.example.org";
|
||||
description = ''
|
||||
The URL the forge reaches this controller at to deliver webhooks,
|
||||
which the daemon registers the swarm's hooks against
|
||||
(`SWARM_CONTROLLER_PUBLIC_URL`). It names the vhost that fronts the
|
||||
controller, not the controller: the apex gateway on this host answers
|
||||
it whether the controller is behind its socket or its TCP listener.
|
||||
Set it when another URL fronts the controller.
|
||||
|
||||
`null` registers no hooks.
|
||||
'';
|
||||
};
|
||||
|
||||
tcp = {
|
||||
listenAddress = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "0.0.0.0";
|
||||
example = "10.0.0.5";
|
||||
description = ''
|
||||
Address the controller binds when
|
||||
{option}`services.hyperhive.swarm.controller.tcp.enable` is set. An
|
||||
IPv6 address is written in brackets. Binding a port opens nothing:
|
||||
{option}`openFirewall` decides who can reach it, and the TLS
|
||||
handshake decides who may talk once they do.
|
||||
'';
|
||||
};
|
||||
|
||||
openFirewall = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = ''
|
||||
Open {option}`services.hyperhive.swarm.controller.tcp.port` in this
|
||||
host's firewall. Off by default: whether the controller is reachable
|
||||
from off-host is the operator's decision, and a gateway on this same
|
||||
host needs nothing opened. Turn it on when the gateway that fronts
|
||||
the controller runs elsewhere.
|
||||
'';
|
||||
};
|
||||
|
||||
tlsDir = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
readOnly = true;
|
||||
default = "/var/lib/swarm-controller-tls";
|
||||
description = ''
|
||||
Where `swarm-controller-tls.service` writes the server leaf and key
|
||||
the listener serves: `cert.pem` and `key.pem`, the key readable by
|
||||
the controller's group alone. The daemon re-reads both when they
|
||||
change, so a rotation needs no restart.
|
||||
'';
|
||||
};
|
||||
|
||||
gatewayClientCertFile = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
readOnly = true;
|
||||
default = "${hiveTls.stateDir}/controller-client.pem";
|
||||
defaultText = lib.literalExpression ''"''${deploy.hive-controller.tls.stateDir}/controller-client.pem"'';
|
||||
description = ''
|
||||
The client certificate the gateway on this host presents to the
|
||||
controller's listener. `swarm-controller-gateway-client.service`
|
||||
issues it from the store, and the gateway's controller locations
|
||||
name it in `proxy_ssl_certificate`.
|
||||
'';
|
||||
};
|
||||
|
||||
gatewayClientKeyFile = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
readOnly = true;
|
||||
default = "${hiveTls.stateDir}/controller-client-key.pem";
|
||||
defaultText = lib.literalExpression ''"''${deploy.hive-controller.tls.stateDir}/controller-client-key.pem"'';
|
||||
description = ''
|
||||
Private key for {option}`gatewayClientCertFile`, mode `0640` and group
|
||||
`nginx`, the way the gateway's own key is readable.
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
swarmctlEnvironment = lib.mkOption {
|
||||
type = lib.types.attrsOf lib.types.str;
|
||||
default = { };
|
||||
|
|
@ -751,12 +927,35 @@ in
|
|||
in both options.
|
||||
'';
|
||||
}
|
||||
{
|
||||
assertion = !tcpCfg.enable || (tcpCfg.domain != null && tcpCfg.clientName != null);
|
||||
message = ''
|
||||
services.hyperhive.swarm.controller.tcp.enable needs
|
||||
services.hyperhive.swarm.controller.tcp.domain, the name the gateway
|
||||
dials the controller by, and services.hyperhive.swarm.domain, which
|
||||
the gateway's client identity is named under. Set both, or leave the
|
||||
TCP listener off.
|
||||
'';
|
||||
}
|
||||
{
|
||||
assertion = !tcpCfg.enable || caTrust.useSelfSigned;
|
||||
message = ''
|
||||
services.hyperhive.swarm.controller.tcp.enable needs the self-signed
|
||||
gateway mode: the controller trusts the secret store's services root
|
||||
for client certificates, and only a self-signed gateway host keeps
|
||||
that root on disk. Leave the TCP listener off where the gateway
|
||||
serves an operator certificate or ACME.
|
||||
'';
|
||||
}
|
||||
];
|
||||
|
||||
systemd.services.swarm-controller = {
|
||||
description = "hyperhive swarm-level controller daemon";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network.target" ];
|
||||
# The listener's certificate files must exist when the daemon opens it. A
|
||||
# start that finds them missing fails and `Restart=` tries again.
|
||||
after = [ "network.target" ] ++ lib.optional tcpOn "swarm-controller-tls.service";
|
||||
wants = lib.optional tcpOn "swarm-controller-tls.service";
|
||||
|
||||
serviceConfig = {
|
||||
ExecStart = "${deployCfg.swarm-controller.package}/bin/swarm-controller";
|
||||
|
|
@ -901,7 +1100,113 @@ in
|
|||
// swarmNameEnv
|
||||
// matrixHomeserverEnv
|
||||
// baoEnv
|
||||
// metaEnv;
|
||||
// metaEnv
|
||||
// tcpEnv;
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = lib.mkIf (tcpOn && tcpHost.openFirewall) [ tcpCfg.port ];
|
||||
|
||||
# The listener's server leaf, from the store's `${tcpCfg.serverPkiRole}`
|
||||
# role under the controller's own store identity — the one it already logs
|
||||
# in with, whose policy gains this one issue path when the listener is on.
|
||||
# Same shape as the queue's leaf unit (./swarm-nats.nix,
|
||||
# `swarm-bao-nats-tls`): the daemon re-reads the files, so rotating needs
|
||||
# no restart, which for this daemon would drop its job graph.
|
||||
systemd.services.swarm-controller-tls = lib.mkIf tcpOn {
|
||||
description = "Issue the swarm-controller's TCP listener certificate from the secret store's PKI";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [
|
||||
"container@${baoCfg.machine}.service"
|
||||
"swarm-bao-pki.service"
|
||||
"swarm-bao-controller-policy.service"
|
||||
];
|
||||
path = [
|
||||
deployCfg.bao.package
|
||||
pkgs.jq
|
||||
pkgs.openssl
|
||||
pkgs.coreutils
|
||||
pkgs.gnugrep
|
||||
];
|
||||
# The store is sealed or unreachable for as long as it takes an operator
|
||||
# to unseal it; see ./lib/store-retry.nix.
|
||||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||||
serviceConfig = storeRetry.serviceConfig // {
|
||||
Type = "oneshot";
|
||||
UMask = "0077";
|
||||
SyslogIdentifier = "swarm-controller-tls";
|
||||
TimeoutStartSec = "5min";
|
||||
};
|
||||
environment = {
|
||||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
BAO_CLIENT_CERT = deployCfg.swarm-controller.baoClientCertFile;
|
||||
BAO_CLIENT_KEY = deployCfg.swarm-controller.baoClientKeyFile;
|
||||
}
|
||||
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
|
||||
BAO_CACERT = deployCfg.bao.serverCaFile;
|
||||
};
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
d=${lib.escapeShellArg tcpHost.tlsDir}
|
||||
name=${lib.escapeShellArg tcpCfg.domain}
|
||||
install -d -m 0750 -g swarm-controller "$d"
|
||||
|
||||
# Missing, past half its window, or naming something other than the
|
||||
# configured domain.
|
||||
reissue=0
|
||||
{ [ -s "$d/cert.pem" ] && [ -s "$d/key.pem" ]; } || reissue=1
|
||||
openssl x509 -in "$d/cert.pem" -noout -checkend ${
|
||||
toString (deployCfg.bao.servicesPkiLeafTtlHours * 3600 / 2)
|
||||
} >/dev/null 2>&1 || reissue=1
|
||||
openssl x509 -in "$d/cert.pem" -noout -checkhost "$name" 2>/dev/null | grep -q ' does match ' || reissue=1
|
||||
if [ "$reissue" = 0 ]; then
|
||||
echo "controller listener leaf valid for $name — leaving it alone"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
err="$(mktemp)"
|
||||
resp="$(mktemp "$d/issue.json.XXXXXX")"
|
||||
trap 'rm -f "$err" "$resp"' EXIT
|
||||
|
||||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||||
echo "could not log in to the swarm secret store with the controller's certificate." >&2
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
export BAO_TOKEN
|
||||
|
||||
echo "requesting the controller's listener leaf for $name from the store"
|
||||
if ! bao write -format=json \
|
||||
${lib.escapeShellArg "${deployCfg.bao.servicesPkiMountPath}/issue/${tcpCfg.serverPkiRole}"} \
|
||||
common_name="$name" > "$resp" 2>"$err"; then
|
||||
echo "the store refused to issue the controller's listener certificate." >&2
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
jq -r '.data.private_key // empty' < "$resp" > "$d/key.pem.new"
|
||||
jq -r '.data.certificate // empty' < "$resp" > "$d/cert.pem.new"
|
||||
for f in "$d/key.pem.new" "$d/cert.pem.new"; do
|
||||
if [ ! -s "$f" ]; then
|
||||
echo "the store's response was missing a field: $f is empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
chgrp swarm-controller "$d/key.pem.new" "$d/cert.pem.new"
|
||||
chmod 0640 "$d/key.pem.new" "$d/cert.pem.new"
|
||||
# Key first: the daemon pairs whatever it reads, and a new key beside
|
||||
# the old certificate is a pair it refuses and retries.
|
||||
mv -f "$d/key.pem.new" "$d/key.pem"
|
||||
mv -f "$d/cert.pem.new" "$d/cert.pem"
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.timers.swarm-controller-tls = lib.mkIf tcpOn {
|
||||
description = "Daily renewal check for the swarm-controller's TCP listener certificate";
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnCalendar = "daily";
|
||||
Persistent = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue