Watch
0
0
Fork
You've already forked hyperhive
0

gateway reaches the controller over tls with its own client leaf when the tcp listener is on

This commit is contained in:
atlas 2026-11-15 16:10:59 +01:00 • committed by mara
commit 187ed82de0
8 changed files with 768 additions and 49 deletions

View file

@ -291,6 +291,22 @@ in
''; '';
}; };
verifiedProxyToWithClientCert = lib.mkOption {
type = lib.types.functionTo (lib.types.functionTo lib.types.lines);
internal = true;
readOnly = true;
description = ''
Read-only: `name -> { certificate, key } -> lines`. The
`verifiedProxyTo` lines plus `proxy_ssl_certificate` and
`proxy_ssl_certificate_key`, for an upstream that requires the
gateway to present a client certificate. `certificate` and `key` are
host paths nginx can read (the key `0640` to the `nginx` group).
Separate from `verifiedProxyTo` so that every existing caller keeps
rendering exactly what it rendered.
'';
};
securityHeaders = lib.mkOption { securityHeaders = lib.mkOption {
type = lib.types.lines; type = lib.types.lines;
internal = true; internal = true;

View file

@ -45,6 +45,39 @@ let
sslCertificateKey = tlsKey; sslCertificateKey = tlsKey;
}; };
# Dial another service on this hive BY NAME over https, verified.
# nginx verifies nothing by default (`proxy_ssl_verify` is off), so a
# `proxy_pass https://…` without this is encrypted but unauthenticated
# — invisibly, it works and keeps working against any certificate at
# all. Full reasoning for every directive here (plus two real
# footguns — session-cache keying, and a `Host`-header clobber that
# can recurse a subrequest into itself) is in docs/4-implementation/networking/gateway.md's
# "Dialing another vhost by name" section — read it before touching
# this.
verifiedProxyTo = name: ''
proxy_ssl_verify on;
proxy_ssl_verify_depth 3;
proxy_ssl_trusted_certificate ${caBundle};
proxy_ssl_name ${name};
proxy_ssl_server_name on;
proxy_set_header Host ${name};
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Server $hostname;
'';
# `verifiedProxyTo`, plus the client certificate nginx presents to an
# upstream that demands one (`ssl_verify_client on`). Opt-in per call: the
# plain form sends none, which is what every existing caller wants.
verifiedProxyToWithClientCert =
name:
{ certificate, key }:
verifiedProxyTo name
+ ''
proxy_ssl_certificate ${certificate};
proxy_ssl_certificate_key ${key};
'';
hstsDirectives = lib.concatStringsSep "; " ( hstsDirectives = lib.concatStringsSep "; " (
[ "max-age=${toString cfg.hsts.maxAge}" ] [ "max-age=${toString cfg.hsts.maxAge}" ]
++ lib.optional cfg.hsts.includeSubDomains "includeSubDomains" ++ lib.optional cfg.hsts.includeSubDomains "includeSubDomains"
@ -82,26 +115,7 @@ in
else else
vhostTls; vhostTls;
# Dial another service on this hive BY NAME over https, verified. inherit verifiedProxyTo verifiedProxyToWithClientCert;
# nginx verifies nothing by default (`proxy_ssl_verify` is off), so a
# `proxy_pass https://…` without this is encrypted but unauthenticated
# — invisibly, it works and keeps working against any certificate at
# all. Full reasoning for every directive here (plus two real
# footguns — session-cache keying, and a `Host`-header clobber that
# can recurse a subrequest into itself) is in docs/4-implementation/networking/gateway.md's
# "Dialing another vhost by name" section — read it before touching
# this.
verifiedProxyTo = name: ''
proxy_ssl_verify on;
proxy_ssl_verify_depth 3;
proxy_ssl_trusted_certificate ${caBundle};
proxy_ssl_name ${name};
proxy_ssl_server_name on;
proxy_set_header Host ${name};
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Server $hostname;
'';
# Security headers added at the server scope on every vhost. # Security headers added at the server scope on every vhost.
# nginx's add_header inheritance rule: a location that defines its # nginx's add_header inheritance rule: a location that defines its

View file

@ -292,6 +292,13 @@ let
BAO_CACERT = baoDeploy.serverCaFile; BAO_CACERT = baoDeploy.serverCaFile;
}; };
storeRetry = import ./lib/store-retry.nix { }; storeRetry = import ./lib/store-retry.nix { };
# The controller's TCP listener (./swarm-controller.nix): the gateway that
# fronts the controller presents a client leaf of its own to it. Only the
# host that runs the controller declares that vhost, so only it needs the leaf.
tcpCfg = hyperhiveCfg.swarm.controller.tcp;
swarmControllerCfg = hyperhiveCfg.deploy.swarm-controller;
gatewayClientLeaf = swarmControllerCfg.enable && tcpCfg.enable;
in in
{ {
# Host-side TLS for the self-signed gateway mode. # Host-side TLS for the self-signed gateway mode.
@ -953,6 +960,125 @@ in
}; };
}; };
# The gateway's client leaf for the controller's TCP listener, issued by
# the store's `swarm-controller-gateway-client` role: `clientAuth` only,
# one name, so the swarm-services leaf is not a substitute. Logs in with
# the services-issuer identity this host already holds, whose policy gains
# the one issue path where the listener is on.
#
# nginx names both files in `proxy_ssl_certificate*`, and refuses to start
# on a config naming a missing file, so a pair always exists once this
# unit has run: a throwaway self-signed one the listener refuses stands in
# until the store answers, and the controller's locations answer 502 in
# the meantime instead of the whole gateway staying down.
systemd.services.swarm-controller-gateway-client = lib.mkIf gatewayClientLeaf {
description = "Issue the gateway's client certificate for the swarm-controller's TCP listener";
wantedBy = [ "multi-user.target" ];
before = [ "nginx.service" ];
after = [
"container@${baoCfg.machine}.service"
"swarm-bao-pki.service"
"swarm-bao-services-issuer-policy.service"
];
path = servicesCertPath;
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
UMask = "0077";
SyslogIdentifier = "swarm-controller-gateway-client";
TimeoutStartSec = servicesCertTimeoutStartSec;
};
environment = servicesCertEnvironment;
script = ''
set -euo pipefail
d=${lib.escapeShellArg cfg.stateDir}
cert=${lib.escapeShellArg swarmControllerCfg.tcp.gatewayClientCertFile}
key=${lib.escapeShellArg swarmControllerCfg.tcp.gatewayClientKeyFile}
root="$d/swarm-services-root.pem"
name=${lib.escapeShellArg (toString tcpCfg.clientName)}
install -d -m 0755 "$d"
# Same readability as the gateway's own key: nginx's pre-start config
# test runs as the nginx user.
place() { # $1 = certificate, $2 = key, written in place of the pair
chgrp nginx "$2"
chmod 0640 "$2"
chmod 0644 "$1"
mv -f "$2" "$key"
mv -f "$1" "$cert"
}
if [ ! -s "$cert" ] || [ ! -s "$key" ]; then
echo "no client certificate yet — placing a throwaway one so nginx can load its config"
openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \
-days 1 -subj "/CN=placeholder" -keyout "$key.new" -out "$cert.new" 2>/dev/null
place "$cert.new" "$key.new"
fi
# Missing, past half its window, naming something else, not chaining
# to the store's root, or not usable as a client certificate. The
# throwaway above fails the chain test, which is what replaces it.
reissue=0
openssl x509 -in "$cert" -noout -checkend ${
toString (baoDeploy.servicesPkiLeafTtlHours * 3600 / 2)
} >/dev/null 2>&1 || reissue=1
openssl x509 -in "$cert" -noout -checkhost "$name" 2>/dev/null | grep -q ' does match ' || reissue=1
{ [ -s "$root" ] && openssl verify -CAfile "$root" -purpose sslclient "$cert" >/dev/null 2>&1; } || reissue=1
if [ "$reissue" = 0 ]; then
echo "gateway client leaf valid for $name — leaving it alone"
exit 0
fi
${requireServicesIssuer "ask the store for the gateway's client leaf"}
err="$(mktemp)"
resp="$(mktemp "$d/controller-client.json.XXXXXX")"
trap 'rm -f "$err" "$resp"' EXIT
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to the swarm secret store with this host's services-issuer certificate." >&2
cat "$err" >&2
exit 1
fi
export BAO_TOKEN
echo "requesting the gateway's client leaf for $name from the store"
if ! bao write -format=json \
${lib.escapeShellArg "${baoServicesPkiMount}/issue/${tcpCfg.clientPkiRole}"} \
common_name="$name" > "$resp" 2>"$err"; then
echo "the store refused to issue the gateway's client certificate." >&2
cat "$err" >&2
exit 1
fi
jq -r '.data.private_key // empty' < "$resp" > "$key.new"
jq -r '.data.certificate // empty' < "$resp" > "$cert.new"
for f in "$key.new" "$cert.new"; do
if [ ! -s "$f" ]; then
echo "the store's response was missing a field: $f is empty" >&2
exit 1
fi
done
place "$cert.new" "$key.new"
# `--no-block` for the reason `swarm-services-cert` gives: this unit
# is ordered before nginx, so a blocking reload would wait on us.
if systemctl is-active --quiet nginx.service; then
echo "gateway client leaf rotated — reloading nginx"
systemctl reload --no-block nginx.service
fi
'';
};
systemd.timers.swarm-controller-gateway-client = lib.mkIf gatewayClientLeaf {
description = "Daily renewal check for the gateway's controller client certificate";
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = "daily";
Persistent = true;
};
};
# Daily rather than weekly, for the reason # Daily rather than weekly, for the reason
# ./swarm-nats.nix's `swarm-bao-nats-tls` timer is: this leaf needs the # ./swarm-nats.nix's `swarm-bao-nats-tls` timer is: this leaf needs the
# store, which can be sealed or unreachable, so there is a retry per day # store, which can be sealed or unreachable, so there is a retry per day

View file

@ -650,6 +650,13 @@ let
path "${sshAgentsMountPath}/config/ca" { path "${sshAgentsMountPath}/config/ca" {
capabilities = ["read"] capabilities = ["read"]
} }
''
# The controller's TCP listener leaf: one issue path on a role of its own,
# present only where the listener is on.
+ lib.optionalString tcpCfg.enable ''
path "${servicesPkiMountPath}/issue/${tcpCfg.serverPkiRole}" {
capabilities = ["update"]
}
''; '';
# The identity authelia's mint creates its OIDC client secrets in the store # The identity authelia's mint creates its OIDC client secrets in the store
@ -979,6 +986,10 @@ let
# and has to spell it the same way. # and has to spell it the same way.
servicesPkiMountPath = baoDeploy.servicesPkiMountPath; servicesPkiMountPath = baoDeploy.servicesPkiMountPath;
# The controller's optional TCP listener: its two roles are written here, and
# its two principals are granted the matching issue paths below.
tcpCfg = hyperhiveCfg.swarm.controller.tcp;
# Subject of the root generated into that mount. A label for a human reading # Subject of the root generated into that mount. A label for a human reading
# a chain, not an identity anything authenticates against. Falls through so # a chain, not an identity anything authenticates against. Falls through so
# a hive that has set neither a swarm name nor a domain still evaluates. # a hive that has set neither a swarm name nor a domain still evaluates.
@ -1015,6 +1026,38 @@ let
servicesPkiLeafTtl = "${toString servicesPkiLeafTtlHours}h"; servicesPkiLeafTtl = "${toString servicesPkiLeafTtlHours}h";
servicesPkiLeafTtlSeconds = servicesPkiLeafTtlHours * 3600; servicesPkiLeafTtlSeconds = servicesPkiLeafTtlHours * 3600;
# The two roles, as script to splice after the services role's write.
# Empty where the listener is off, so the unit's script is the one it was
# before. Same narrowing as the services role, one name each:
# * `serverPkiRole` issues the listener's server leaf for `tcp.domain`
# alone, `serverAuth` only.
# * `clientPkiRole` issues the gateway's client leaf for `tcp.clientName`
# alone, `clientAuth` only. The opposite flag pair from the services
# role, which is why the services leaf cannot present itself to the
# listener.
tcpRoleWrite = role: name: flags: ''
bao write ${lib.escapeShellArg "${servicesPkiMountPath}/roles/${role}"} \
allowed_domains=${lib.escapeShellArg (toString name)} \
allow_bare_domains=true \
allow_subdomains=false \
allow_glob_domains=false \
allow_localhost=false \
allow_any_name=false \
allow_ip_sans=false \
enforce_hostnames=true \
${flags} \
key_type=rsa \
key_bits=4096 \
ttl=${servicesPkiLeafTtl} \
max_ttl=${servicesPkiLeafTtl}
'';
tcpRoles = lib.optionalString tcpCfg.enable (
"\n\n"
+ tcpRoleWrite tcpCfg.serverPkiRole tcpCfg.domain "server_flag=true client_flag=false"
+ "\n"
+ tcpRoleWrite tcpCfg.clientPkiRole tcpCfg.clientName "server_flag=false client_flag=true"
);
# The role every swarm-service certificate will be issued through, and the # The role every swarm-service certificate will be issued through, and the
# whole narrowing of this mount. A role rather than a name-constrained # whole narrowing of this mount. A role rather than a name-constrained
# sub-CA: the engine enforces "these names and no others" at issue time, so # sub-CA: the engine enforces "these names and no others" at issue time, so
@ -1049,6 +1092,15 @@ let
path "${servicesPkiMountPath}/issue/${servicesPkiRoleName}" { path "${servicesPkiMountPath}/issue/${servicesPkiRoleName}" {
capabilities = ["update"] capabilities = ["update"]
} }
''
# The gateway's client leaf for the controller's TCP listener. Every gateway
# host holds this identity, so every gateway host can obtain the leaf: what the
# controller trusts is "a gateway of this swarm". The role issues one name,
# with `clientAuth` only.
+ lib.optionalString tcpCfg.enable ''
path "${servicesPkiMountPath}/issue/${tcpCfg.clientPkiRole}" {
capabilities = ["update"]
}
''; '';
# The queue's principal: one `update` on its own role, the shape of the # The queue's principal: one `update` on its own role, the shape of the
@ -4057,7 +4109,7 @@ in
key_type=rsa \ key_type=rsa \
key_bits=4096 \ key_bits=4096 \
ttl=${servicesPkiLeafTtl} \ ttl=${servicesPkiLeafTtl} \
max_ttl=${servicesPkiLeafTtl} max_ttl=${servicesPkiLeafTtl}${tcpRoles}
# `certificate=` is the CA, so this role trusts every leaf that # `certificate=` is the CA, so this role trusts every leaf that
# CA signed and `allowed_common_names` is the whole narrowing — # CA signed and `allowed_common_names` is the whole narrowing —

View file

@ -7,6 +7,13 @@
# nginx is the only intended client, it runs on this same host and so # nginx is the only intended client, it runs on this same host and so
# reaches the socket by path, and a socket that is never bound to an # reaches the socket by path, and a socket that is never bound to an
# address cannot be reached from off-host by mistake. # address cannot be reached from off-host by mistake.
#
# `swarm.controller.tcp.enable` adds a TLS listener beside the socket, which the
# gateway then dials instead of the socket. The controller checks nobody per
# request, so the listener completes a handshake for one identity only: a
# client certificate the swarm's store issued for the gateway, with the
# `clientAuth` usage. ./swarm-ui.nix still declares the vhost on the host that
# runs this daemon, so the gateway stays on that host.
{ {
pkgs, pkgs,
lib, lib,
@ -137,6 +144,16 @@ let
swarmDomain = config.services.hyperhive.swarm.domain; swarmDomain = config.services.hyperhive.swarm.domain;
matrixCfg = config.services.hyperhive.swarm.matrix; matrixCfg = config.services.hyperhive.swarm.matrix;
# The optional TCP listener. `tcpCfg` is the swarm-wide half, evaluated
# identically on every host: the store's host writes its roles from it, the
# gateway's host issues its client leaf from it, and this host serves it.
# `tcpHost` is what only this host decides.
tcpCfg = cfg.tcp;
tcpHost = deployCfg.swarm-controller.tcp;
hiveTls = deployCfg.hive-controller.tls;
storeRetry = import ./lib/store-retry.nix { };
tcpOn = deployCfg.swarm-controller.enable && tcpCfg.enable;
# Which swarm `swarmctl login` logs in to when given none: this host's own. # Which swarm `swarmctl login` logs in to when given none: this host's own.
# Read by `login` alone. `SWARMCTL_SWARM` would also move every socket verb # Read by `login` alone. `SWARMCTL_SWARM` would also move every socket verb
# onto the gateway, where each needs a login. A default, so an operator's # onto the gateway, where each needs a login. A default, so an operator's
@ -215,28 +232,29 @@ let
}; };
# How the forge must address this controller to deliver a swarm-wide # How the forge must address this controller to deliver a swarm-wide
# webhook. ./swarm-ui.nix declares the vhost and the `/webhook/forge/` # webhook: `deploy.swarm-controller.publicUrl`. ./swarm-ui.nix declares the
# location inside it on every host that runs this daemon, so the only gate # vhost and the `/webhook/forge/` location inside it on every host that runs
# left is a swarm domain to name it by. # this daemon, so the only gate left is a URL to name it by.
# #
# 🔑 The gate is the point, not a detail. The daemon registers hooks only # 🔑 The gate is the point, not a detail. The daemon registers hooks only
# when this is set, and a hook whose `target_url` nothing answers is worse # when this is set, and a hook whose `target_url` nothing answers is worse
# than no hook at all — forgejo keeps the registration, marks every # than no hook at all — forgejo keeps the registration, marks every
# delivery failed, and the hook still reads as configured. # delivery failed, and the hook still reads as configured.
# webhookEnv = lib.optionalAttrs (deployCfg.swarm-controller.publicUrl != null) {
# ⚠️ Correct ONLY while the vhost and this daemon share a host, and they do SWARM_CONTROLLER_PUBLIC_URL = deployCfg.swarm-controller.publicUrl;
# by construction: the UI's `/api/` location proxies };
# `http://unix:<socketPath>`, a path that resolves nowhere else, and this
# daemon binds no TCP address at all (see the header). # What the daemon needs to open its TCP listener (`swarm-controller/src/
# # tcp_tls.rs`). All five or none: the daemon refuses a partial set. The
# ⇒ The day this daemon grows a TCP listener so another host can front it, # client CA is the store's services root, the one the gateway's client leaf
# THIS LINE SILENTLY STOPS BEING RIGHT: hooks get registered against a vhost # chains to, from the file `hive-tls-ca` already keeps on every self-signed
# that may no longer be the one fronting the daemon, and nothing errors. That is # gateway host.
# the moment to add an explicit `publicUrl` option — not before, because tcpEnv = lib.optionalAttrs tcpOn {
# until then there is exactly one derivable answer and an option would only SWARM_CONTROLLER_TCP_LISTEN = "${tcpHost.listenAddress}:${toString tcpCfg.port}";
# be a second place to get it wrong. SWARM_CONTROLLER_TCP_CERT = "${tcpHost.tlsDir}/cert.pem";
webhookEnv = lib.optionalAttrs (swarmDomain != null) { SWARM_CONTROLLER_TCP_KEY = "${tcpHost.tlsDir}/key.pem";
SWARM_CONTROLLER_PUBLIC_URL = "https://${swarmDomain}"; SWARM_CONTROLLER_TCP_CLIENT_CA = "${hiveTls.stateDir}/swarm-services-root.pem";
SWARM_CONTROLLER_TCP_CLIENT_NAME = tcpCfg.clientName;
}; };
# Not a secret to deliver — `swarm-authelia-bridge`'s own bearer check # Not a secret to deliver — `swarm-authelia-bridge`'s own bearer check
@ -396,6 +414,84 @@ in
''; '';
}; };
# Swarm tier, not `deploy.*`: the store's host writes the two roles below,
# the host fronting the controller issues the gateway's client leaf, and the
# controller's host serves — three hosts that must agree on one set of
# names, so each reads them from the option every host evaluates alike.
tcp = {
enable = lib.mkEnableOption ''
the controller's TCP listener, TLS with a required client certificate.
Off: the controller serves its unix socket only, which is what the
gateway on the same host reaches it through.
On: the gateway's controller locations dial {option}`domain` over TLS
and present a client leaf, and the controller accepts that one
identity. The unix socket keeps working beside it. The listener
reaches only the gateway: the controller trusts its transport and
checks nobody per request, so a certificate the swarm root signed for
anything else is refused at the handshake.
Set it identically on every host of the swarm. The store's host
writes the two roles from it, the controller's host issues its server
leaf, and the host fronting the controller issues the gateway's client
leaf, each through an identity it already holds.
'';
domain = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "controller.example.org";
description = ''
The name the gateway dials the controller by. It becomes the server
certificate's name (the store's `swarm-controller-tls` role issues
for this name alone) and what nginx verifies the controller against.
It must resolve, from the gateway's host, to an address the
controller listens on. Required when {option}`enable` is set.
'';
};
port = lib.mkOption {
type = lib.types.port;
default = 9443;
description = "TCP port the controller listens on and the gateway dials.";
};
clientName = lib.mkOption {
type = lib.types.nullOr lib.types.str;
readOnly = true;
default = if swarmDomain == null then null else "gateway-client.${swarmDomain}";
defaultText = lib.literalExpression ''"gateway-client.''${services.hyperhive.swarm.domain}"'';
description = ''
The one name a client certificate must carry for the controller to
accept it. The store's `swarm-controller-gateway-client` role issues for this
name alone, with the `clientAuth` usage and not `serverAuth`, so the
swarm-services leaf the gateway serves browsers with does not
qualify.
'';
};
serverPkiRole = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "swarm-controller-tls";
description = ''
The store's `pki` role that issues the controller's server leaf.
Starts with `swarm-`: the granter that writes it may write roles
under that prefix only.
'';
};
clientPkiRole = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "swarm-controller-gateway-client";
description = ''
The store's `pki` role that issues the gateway's client leaf. Starts
with `swarm-`, as {option}`serverPkiRole` does.
'';
};
};
}; };
# What stays above is what the controller IS to the swarm: # What stays above is what the controller IS to the swarm:
@ -610,6 +706,86 @@ in
''; '';
}; };
publicUrl = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = if swarmDomain == null then null else "https://${swarmDomain}";
defaultText = lib.literalExpression ''"https://''${services.hyperhive.swarm.domain}"'';
example = "https://swarm.example.org";
description = ''
The URL the forge reaches this controller at to deliver webhooks,
which the daemon registers the swarm's hooks against
(`SWARM_CONTROLLER_PUBLIC_URL`). It names the vhost that fronts the
controller, not the controller: the apex gateway on this host answers
it whether the controller is behind its socket or its TCP listener.
Set it when another URL fronts the controller.
`null` registers no hooks.
'';
};
tcp = {
listenAddress = lib.mkOption {
type = lib.types.str;
default = "0.0.0.0";
example = "10.0.0.5";
description = ''
Address the controller binds when
{option}`services.hyperhive.swarm.controller.tcp.enable` is set. An
IPv6 address is written in brackets. Binding a port opens nothing:
{option}`openFirewall` decides who can reach it, and the TLS
handshake decides who may talk once they do.
'';
};
openFirewall = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Open {option}`services.hyperhive.swarm.controller.tcp.port` in this
host's firewall. Off by default: whether the controller is reachable
from off-host is the operator's decision, and a gateway on this same
host needs nothing opened. Turn it on when the gateway that fronts
the controller runs elsewhere.
'';
};
tlsDir = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "/var/lib/swarm-controller-tls";
description = ''
Where `swarm-controller-tls.service` writes the server leaf and key
the listener serves: `cert.pem` and `key.pem`, the key readable by
the controller's group alone. The daemon re-reads both when they
change, so a rotation needs no restart.
'';
};
gatewayClientCertFile = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "${hiveTls.stateDir}/controller-client.pem";
defaultText = lib.literalExpression ''"''${deploy.hive-controller.tls.stateDir}/controller-client.pem"'';
description = ''
The client certificate the gateway on this host presents to the
controller's listener. `swarm-controller-gateway-client.service`
issues it from the store, and the gateway's controller locations
name it in `proxy_ssl_certificate`.
'';
};
gatewayClientKeyFile = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "${hiveTls.stateDir}/controller-client-key.pem";
defaultText = lib.literalExpression ''"''${deploy.hive-controller.tls.stateDir}/controller-client-key.pem"'';
description = ''
Private key for {option}`gatewayClientCertFile`, mode `0640` and group
`nginx`, the way the gateway's own key is readable.
'';
};
};
swarmctlEnvironment = lib.mkOption { swarmctlEnvironment = lib.mkOption {
type = lib.types.attrsOf lib.types.str; type = lib.types.attrsOf lib.types.str;
default = { }; default = { };
@ -751,12 +927,35 @@ in
in both options. in both options.
''; '';
} }
{
assertion = !tcpCfg.enable || (tcpCfg.domain != null && tcpCfg.clientName != null);
message = ''
services.hyperhive.swarm.controller.tcp.enable needs
services.hyperhive.swarm.controller.tcp.domain, the name the gateway
dials the controller by, and services.hyperhive.swarm.domain, which
the gateway's client identity is named under. Set both, or leave the
TCP listener off.
'';
}
{
assertion = !tcpCfg.enable || caTrust.useSelfSigned;
message = ''
services.hyperhive.swarm.controller.tcp.enable needs the self-signed
gateway mode: the controller trusts the secret store's services root
for client certificates, and only a self-signed gateway host keeps
that root on disk. Leave the TCP listener off where the gateway
serves an operator certificate or ACME.
'';
}
]; ];
systemd.services.swarm-controller = { systemd.services.swarm-controller = {
description = "hyperhive swarm-level controller daemon"; description = "hyperhive swarm-level controller daemon";
wantedBy = [ "multi-user.target" ]; wantedBy = [ "multi-user.target" ];
after = [ "network.target" ]; # The listener's certificate files must exist when the daemon opens it. A
# start that finds them missing fails and `Restart=` tries again.
after = [ "network.target" ] ++ lib.optional tcpOn "swarm-controller-tls.service";
wants = lib.optional tcpOn "swarm-controller-tls.service";
serviceConfig = { serviceConfig = {
ExecStart = "${deployCfg.swarm-controller.package}/bin/swarm-controller"; ExecStart = "${deployCfg.swarm-controller.package}/bin/swarm-controller";
@ -901,7 +1100,113 @@ in
// swarmNameEnv // swarmNameEnv
// matrixHomeserverEnv // matrixHomeserverEnv
// baoEnv // baoEnv
// metaEnv; // metaEnv
// tcpEnv;
};
networking.firewall.allowedTCPPorts = lib.mkIf (tcpOn && tcpHost.openFirewall) [ tcpCfg.port ];
# The listener's server leaf, from the store's `${tcpCfg.serverPkiRole}`
# role under the controller's own store identity — the one it already logs
# in with, whose policy gains this one issue path when the listener is on.
# Same shape as the queue's leaf unit (./swarm-nats.nix,
# `swarm-bao-nats-tls`): the daemon re-reads the files, so rotating needs
# no restart, which for this daemon would drop its job graph.
systemd.services.swarm-controller-tls = lib.mkIf tcpOn {
description = "Issue the swarm-controller's TCP listener certificate from the secret store's PKI";
wantedBy = [ "multi-user.target" ];
after = [
"container@${baoCfg.machine}.service"
"swarm-bao-pki.service"
"swarm-bao-controller-policy.service"
];
path = [
deployCfg.bao.package
pkgs.jq
pkgs.openssl
pkgs.coreutils
pkgs.gnugrep
];
# The store is sealed or unreachable for as long as it takes an operator
# to unseal it; see ./lib/store-retry.nix.
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
UMask = "0077";
SyslogIdentifier = "swarm-controller-tls";
TimeoutStartSec = "5min";
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = deployCfg.swarm-controller.baoClientCertFile;
BAO_CLIENT_KEY = deployCfg.swarm-controller.baoClientKeyFile;
}
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
BAO_CACERT = deployCfg.bao.serverCaFile;
};
script = ''
set -euo pipefail
d=${lib.escapeShellArg tcpHost.tlsDir}
name=${lib.escapeShellArg tcpCfg.domain}
install -d -m 0750 -g swarm-controller "$d"
# Missing, past half its window, or naming something other than the
# configured domain.
reissue=0
{ [ -s "$d/cert.pem" ] && [ -s "$d/key.pem" ]; } || reissue=1
openssl x509 -in "$d/cert.pem" -noout -checkend ${
toString (deployCfg.bao.servicesPkiLeafTtlHours * 3600 / 2)
} >/dev/null 2>&1 || reissue=1
openssl x509 -in "$d/cert.pem" -noout -checkhost "$name" 2>/dev/null | grep -q ' does match ' || reissue=1
if [ "$reissue" = 0 ]; then
echo "controller listener leaf valid for $name — leaving it alone"
exit 0
fi
err="$(mktemp)"
resp="$(mktemp "$d/issue.json.XXXXXX")"
trap 'rm -f "$err" "$resp"' EXIT
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to the swarm secret store with the controller's certificate." >&2
cat "$err" >&2
exit 1
fi
export BAO_TOKEN
echo "requesting the controller's listener leaf for $name from the store"
if ! bao write -format=json \
${lib.escapeShellArg "${deployCfg.bao.servicesPkiMountPath}/issue/${tcpCfg.serverPkiRole}"} \
common_name="$name" > "$resp" 2>"$err"; then
echo "the store refused to issue the controller's listener certificate." >&2
cat "$err" >&2
exit 1
fi
jq -r '.data.private_key // empty' < "$resp" > "$d/key.pem.new"
jq -r '.data.certificate // empty' < "$resp" > "$d/cert.pem.new"
for f in "$d/key.pem.new" "$d/cert.pem.new"; do
if [ ! -s "$f" ]; then
echo "the store's response was missing a field: $f is empty" >&2
exit 1
fi
done
chgrp swarm-controller "$d/key.pem.new" "$d/cert.pem.new"
chmod 0640 "$d/key.pem.new" "$d/cert.pem.new"
# Key first: the daemon pairs whatever it reads, and a new key beside
# the old certificate is a pair it refuses and retries.
mv -f "$d/key.pem.new" "$d/key.pem"
mv -f "$d/cert.pem.new" "$d/cert.pem"
'';
};
systemd.timers.swarm-controller-tls = lib.mkIf tcpOn {
description = "Daily renewal check for the swarm-controller's TCP listener certificate";
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = "daily";
Persistent = true;
};
}; };
}; };
} }

View file

@ -106,12 +106,46 @@ let
matrixCfg = swarmCfg.matrix; matrixCfg = swarmCfg.matrix;
serverName = if matrixCfg.serverName != null then matrixCfg.serverName else apex; serverName = if matrixCfg.serverName != null then matrixCfg.serverName else apex;
servesMatrixDiscovery = matrixCfg.gatewayHost != null && serverName == apex; servesMatrixDiscovery = matrixCfg.gatewayHost != null && serverName == apex;
# How the gateway reaches the controller. Its unix socket by default. With
# the controller's TCP listener on, over TLS by the listener's name, with
# the gateway's client leaf — the listener refuses a handshake without it
# (./swarm-controller.nix). `controllerDial` is empty on the socket, so the
# two locations below render exactly what they rendered before the
# listener existed.
# `toString` below so an unset `domain` reaches the assertion that names it
# instead of failing evaluation first.
controllerTcp = swarmCfg.controller.tcp;
controllerUpstream =
if controllerTcp.enable then
"https://${toString controllerTcp.domain}:${toString controllerTcp.port}"
else
"http://unix:${deployCfg.swarm-controller.socketPath}:";
# `proxy_http_version` and `Connection` because `recommendedProxySettings`,
# which sets them, is off on a location that pins its own `Host`: without
# them nginx speaks HTTP/1.0 to the upstream, which ends the SSE streams'
# keep-alive.
controllerDial = lib.optionalString controllerTcp.enable ''
${gatewayCfg.lib.verifiedProxyToWithClientCert (toString controllerTcp.domain) {
certificate = deployCfg.swarm-controller.tcp.gatewayClientCertFile;
key = deployCfg.swarm-controller.tcp.gatewayClientKeyFile;
}}
proxy_http_version 1.1;
proxy_set_header Connection "";
'';
controllerDialAttrs = lib.optionalAttrs controllerTcp.enable {
# nixpkgs appends its own `Host $host` after `extraConfig`, which would
# override the pinned one — see the comment on verifiedProxyTo in
# hive-gateway/vhost-lib.nix.
recommendedProxySettings = false;
};
in in
{ {
# No `enable` of its own: the UI is on wherever # No `enable` of its own: the UI is on wherever
# `services.hyperhive.deploy.swarm-controller.enable` is, because it reaches # `services.hyperhive.deploy.swarm-controller.enable` is, because it reaches
# that daemon over its unix socket. There is no `domain` option: the UI # that daemon over its unix socket, or over its TCP listener when that is on.
# answers on the swarm apex, which the swarm already declares once. # There is no `domain` option: the UI answers on the swarm apex, which the
# swarm already declares once.
# ⚠️ Neither `services.hyperhive.swarm.ui` nor `deploy.swarm-ui` is a # ⚠️ Neither `services.hyperhive.swarm.ui` nor `deploy.swarm-ui` is a
# namespace for anything but the old spellings: the UI's `package` lives # namespace for anything but the old spellings: the UI's `package` lives
@ -295,8 +329,8 @@ in
# shape as the per-hive dashboard's own `/api/` proxy) so the # shape as the per-hive dashboard's own `/api/` proxy) so the
# path swarm-controller registered a route at is the path # path swarm-controller registered a route at is the path
# nginx forwards, no prefix-stripping to keep in sync by hand. # nginx forwards, no prefix-stripping to keep in sync by hand.
"/api/" = { "/api/" = controllerDialAttrs // {
proxyPass = "http://unix:${deployCfg.swarm-controller.socketPath}:"; proxyPass = controllerUpstream;
# `Remote-User` from authelia's answer, set here so it replaces # `Remote-User` from authelia's answer, set here so it replaces
# any the client sent: swarm-controller takes it as the sender of # any the client sent: swarm-controller takes it as the sender of
# the text the operator types to an agent (`agent_input.rs`). # the text the operator types to an agent (`agent_input.rs`).
@ -313,7 +347,8 @@ in
# must not sit in nginx's buffer. The controller's 15s # must not sit in nginx's buffer. The controller's 15s
# keep-alive keeps them under nginx's 60s default read timeout. # keep-alive keeps them under nginx's 60s default read timeout.
proxy_buffering off; proxy_buffering off;
''; ''
+ controllerDial;
}; };
# A refused request gets the login page only when it asks for a page. # A refused request gets the login page only when it asks for a page.
# The swarm UI calls `/api/` with `fetch` and `EventSource`, and a # The swarm UI calls `/api/` with `fetch` and `EventSource`, and a
@ -356,8 +391,13 @@ in
# carve-out is exactly as wide as the endpoint that justifies it: # carve-out is exactly as wide as the endpoint that justifies it:
# a future `/webhook/<something-else>` does not inherit the # a future `/webhook/<something-else>` does not inherit the
# bypass by living under a shared prefix. # bypass by living under a shared prefix.
"/webhook/forge/" = { "/webhook/forge/" =
proxyPass = "http://unix:${deployCfg.swarm-controller.socketPath}:"; controllerDialAttrs
// {
proxyPass = controllerUpstream;
}
// lib.optionalAttrs controllerTcp.enable {
extraConfig = controllerDial;
}; };
# Swagger UI: same "nginx hosts the themed dist straight from # Swagger UI: same "nginx hosts the themed dist straight from
# the store, only /api/openapi.json is dynamic" shape as the # the store, only /api/openapi.json is dynamic" shape as the

View file

@ -41,6 +41,26 @@ let
deploy.hive-controller.hyperhiveRev = lib.fixedWidthString 40 "b" ""; deploy.hive-controller.hyperhiveRev = lib.fixedWidthString 40 "b" "";
}; };
# `allLocal` with the controller's TCP listener on, the one fixture that has
# the store, the controller and the gateway together: the two roles, the two
# issue grants and the vhost all render from this one option. The firewall is
# opened so the lookup that finds the port is shown to find it. Off is
# `allLocal` itself.
allLocalTcp = hiveWithController {
deploy.singleHostSwarm = true;
swarm.controller.tcp = {
enable = true;
domain = "controller.t.local";
};
deploy.swarm-controller.tcp.openFirewall = true;
};
# The listener asked for with nothing to name it by.
tcpNoDomain = hive {
deploy.swarm-controller.enable = true;
swarm.controller.tcp.enable = true;
};
bare = shared.bareWithController; bare = shared.bareWithController;
# `lib.hasInfix` matches by regex, which refuses a needle that refers to a # `lib.hasInfix` matches by regex, which refuses a needle that refers to a
@ -3016,6 +3036,149 @@ let
mentions "privateListenAddresses holds" mentions "privateListenAddresses holds"
&& !(mentions "while services.hyperhive.gateway.privateListenAddresses is empty"); && !(mentions "while services.hyperhive.gateway.privateListenAddresses is empty");
} }
{
# The off arm of the controller's TCP listener: a host that never set it
# renders what it rendered before it existed. The gateway still reaches
# the controller by socket, nothing TCP-shaped is rendered, and the
# store writes neither role nor grant. The `swarm-services-cert` unit is
# the presence control for the unit lookup, and the socket in
# `proxyPass` for the location lookup.
name = "a swarm without the controller's TCP listener renders the socket-only controller";
ok =
let
m = allLocal;
l = m.services.nginx.virtualHosts."t.local".locations;
socket = "http://unix:${m.services.hyperhive.deploy.swarm-controller.socketPath}:";
env = m.systemd.services.swarm-controller.environment;
policies = m.systemd.services.swarm-bao-controller-policy.script;
issuer = m.systemd.services.swarm-bao-services-issuer-policy.script;
in
l."/api/".proxyPass == socket
&& l."/webhook/forge/".proxyPass == socket
&& !(lib.hasInfix "proxy_ssl_certificate" l."/api/".extraConfig)
&& !(lib.hasInfix "proxy_ssl" l."/webhook/forge/".extraConfig)
&& !(m.systemd.services ? swarm-controller-tls)
&& !(m.systemd.services ? swarm-controller-gateway-client)
&& !(m.systemd.timers ? swarm-controller-tls)
&& !(m.systemd.timers ? swarm-controller-gateway-client)
&& m.systemd.services ? swarm-services-cert
&& !(lib.any (lib.hasPrefix "SWARM_CONTROLLER_TCP_") (lib.attrNames env))
&& !(lib.elem 9443 m.networking.firewall.allowedTCPPorts)
&& !(lib.hasInfix "swarm-controller-tls" policies)
&& !(lib.hasInfix "swarm-controller-gateway-client" policies)
&& !(lib.hasInfix "swarm-controller-gateway-client" issuer)
&& lib.hasInfix "issue/swarm-services" issuer
&& env.SWARM_CONTROLLER_PUBLIC_URL == "https://t.local"
&& m.services.hyperhive.deploy.swarm-controller.tcp.openFirewall == false
&& allLocalTcp.services.hyperhive.deploy.swarm-controller.tcp.openFirewall == true;
}
{
# The on arm. The gateway dials the listener by name over TLS, verified,
# with the client leaf, and keeps the operator gate on `/api/` and the
# webhook's absence of it; the controller is handed all five listener
# variables, naming the services root as the client CA and the gateway's
# one name as the client; the firewall port is open because the fixture
# asked.
name = "the controller's TCP listener is dialled over verified TLS with the gateway's client leaf";
ok =
let
m = allLocalTcp;
l = m.services.nginx.virtualHosts."t.local".locations;
upstream = "https://controller.t.local:9443";
tls = m.services.hyperhive.deploy.hive-controller.tls.stateDir;
env = m.systemd.services.swarm-controller.environment;
dialled =
loc:
lib.all (s: lib.hasInfix s loc.extraConfig) [
"proxy_ssl_verify on;"
"proxy_ssl_name controller.t.local;"
"proxy_ssl_certificate ${tls}/controller-client.pem;"
"proxy_ssl_certificate_key ${tls}/controller-client-key.pem;"
"proxy_set_header Host controller.t.local;"
];
in
l."/api/".proxyPass == upstream
&& l."/webhook/forge/".proxyPass == upstream
&& dialled l."/api/"
&& dialled l."/webhook/forge/"
&& l."/api/".recommendedProxySettings == false
&& l."/webhook/forge/".recommendedProxySettings == false
&& lib.hasInfix "auth_request /__hive_authelia;" l."/api/".extraConfig
&& !(lib.hasInfix "auth_request" l."/webhook/forge/".extraConfig)
&& env.SWARM_CONTROLLER_TCP_LISTEN == "0.0.0.0:9443"
&& env.SWARM_CONTROLLER_TCP_CERT == "/var/lib/swarm-controller-tls/cert.pem"
&& env.SWARM_CONTROLLER_TCP_KEY == "/var/lib/swarm-controller-tls/key.pem"
&& env.SWARM_CONTROLLER_TCP_CLIENT_CA == "${tls}/swarm-services-root.pem"
&& env.SWARM_CONTROLLER_TCP_CLIENT_NAME == "gateway-client.t.local"
&& lib.elem 9443 m.networking.firewall.allowedTCPPorts
&& env.SWARM_CONTROLLER_PUBLIC_URL == "https://t.local"
&& m.systemd.services.swarm-controller-tls.wantedBy == [ "multi-user.target" ]
&& lib.elem "swarm-controller-tls.service" m.systemd.services.swarm-controller.after;
}
{
# The two roles carry opposite flag pairs, which is the whole reason the
# swarm-services leaf cannot present itself to the listener: the client
# role issues `clientAuth` only and the server role `serverAuth` only.
# Each is read out of the one `bao write` that names it, so swapped flags
# fail here and a role that is missing fails on the first clause. The
# grants are split the same way: the controller may issue its server
# leaf and the gateway-side identity its client leaf, and neither may
# issue the other's.
name = "the store writes a server-only and a client-only role and grants each to one principal";
ok =
let
m = allLocalTcp;
script = m.systemd.services.swarm-bao-controller-policy.script;
write =
role: lib.findFirst (c: lib.hasInfix "/roles/${role} " c) "" (lib.splitString "bao write" script);
server = write "swarm-controller-tls";
client = write "swarm-controller-gateway-client";
issuer = m.systemd.services.swarm-bao-services-issuer-policy.script;
in
server != ""
&& client != ""
&& lib.hasInfix "allowed_domains=controller.t.local" server
&& lib.hasInfix "server_flag=true" server
&& lib.hasInfix "client_flag=false" server
&& lib.hasInfix "allowed_domains=gateway-client.t.local" client
&& lib.hasInfix "server_flag=false" client
&& lib.hasInfix "client_flag=true" client
&& lib.hasInfix "allow_any_name=false" client
&& lib.hasInfix "issue/swarm-controller-tls" script
&& !(lib.hasInfix "issue/swarm-controller-gateway-client" script)
&& lib.hasInfix "issue/swarm-controller-gateway-client" issuer
&& !(lib.hasInfix "issue/swarm-controller-tls" issuer)
&& lib.hasInfix "issue/swarm-services" issuer;
}
{
# The gateway's client leaf is checked for the `clientAuth` purpose
# before it is trusted as current, so a leaf issued under the wrong role
# is replaced rather than kept. The throwaway pair that stands in until
# the store answers is what keeps nginx's config loadable.
name = "the gateway's client leaf is issued with the services-issuer identity and placed for nginx";
ok =
let
u = allLocalTcp.systemd.services.swarm-controller-gateway-client;
in
lib.hasInfix "-purpose sslclient" u.script
&& lib.hasInfix "issue/swarm-controller-gateway-client" u.script
&& lib.hasInfix "chgrp nginx" u.script
&& lib.hasInfix "placeholder" u.script
&& lib.elem "nginx.service" u.before
&& u.environment ? BAO_ADDR
&& allLocalTcp.systemd.timers ? swarm-controller-gateway-client;
}
{
name = "the controller's TCP listener is refused at eval with no name to dial it by";
ok =
let
fired = lib.filter (a: !a.assertion) tcpNoDomain.assertions;
in
lib.any (a: lib.hasInfix "swarm.controller.tcp.domain" a.message) fired
&& !(lib.any (a: lib.hasInfix "swarm.controller.tcp.domain" a.message) (
lib.filter (a: !a.assertion) allLocalTcp.assertions
));
}
] ]
# Starting a unit that stays `active (exited)` is a no-op, so a timer aimed # Starting a unit that stays `active (exited)` is a no-op, so a timer aimed
# at one fires and fetches nothing. # at one fires and fetches nothing.

View file

@ -175,7 +175,10 @@ fn read_pair(cert: &Path, key: &Path, provider: &CryptoProvider) -> Result<Arc<C
.key_provider .key_provider
.load_private_key(key) .load_private_key(key)
.context("the private key is not one rustls can sign with")?; .context("the private key is not one rustls can sign with")?;
Ok(Arc::new(CertifiedKey::new(chain, signing))) let pair = CertifiedKey::new(chain, signing);
pair.keys_match()
.context("the private key does not belong to the certificate")?;
Ok(Arc::new(pair))
} }
/// Accepts exactly the certificates `inner` accepts that also name `name`. /// Accepts exactly the certificates `inner` accepts that also name `name`.