From 187ed82de0110d969feb3665f6af9cc7040ae65f Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 15 Nov 2026 16:10:59 +0100 Subject: [PATCH] gateway reaches the controller over tls with its own client leaf when the tcp listener is on --- nix/host-modules/hive-gateway/options.nix | 16 + nix/host-modules/hive-gateway/vhost-lib.nix | 54 +-- nix/host-modules/hive-tls.nix | 126 +++++++ nix/host-modules/swarm-bao.nix | 54 ++- nix/host-modules/swarm-controller.nix | 343 ++++++++++++++++++-- nix/host-modules/swarm-ui.nix | 56 +++- nix/module-eval/core-toggle.nix | 163 ++++++++++ swarm-controller/src/tcp_tls.rs | 5 +- 8 files changed, 768 insertions(+), 49 deletions(-) diff --git a/nix/host-modules/hive-gateway/options.nix b/nix/host-modules/hive-gateway/options.nix index 620d7a4a7..74183bdbd 100644 --- a/nix/host-modules/hive-gateway/options.nix +++ b/nix/host-modules/hive-gateway/options.nix @@ -291,6 +291,22 @@ in ''; }; + verifiedProxyToWithClientCert = lib.mkOption { + type = lib.types.functionTo (lib.types.functionTo lib.types.lines); + internal = true; + readOnly = true; + description = '' + Read-only: `name -> { certificate, key } -> lines`. The + `verifiedProxyTo` lines plus `proxy_ssl_certificate` and + `proxy_ssl_certificate_key`, for an upstream that requires the + gateway to present a client certificate. `certificate` and `key` are + host paths nginx can read (the key `0640` to the `nginx` group). + + Separate from `verifiedProxyTo` so that every existing caller keeps + rendering exactly what it rendered. + ''; + }; + securityHeaders = lib.mkOption { type = lib.types.lines; internal = true; diff --git a/nix/host-modules/hive-gateway/vhost-lib.nix b/nix/host-modules/hive-gateway/vhost-lib.nix index 4581c188b..0a85b0efe 100644 --- a/nix/host-modules/hive-gateway/vhost-lib.nix +++ b/nix/host-modules/hive-gateway/vhost-lib.nix @@ -45,6 +45,39 @@ let sslCertificateKey = tlsKey; }; + # Dial another service on this hive BY NAME over https, verified. + # nginx verifies nothing by default (`proxy_ssl_verify` is off), so a + # `proxy_pass https://…` without this is encrypted but unauthenticated + # — invisibly, it works and keeps working against any certificate at + # all. Full reasoning for every directive here (plus two real + # footguns — session-cache keying, and a `Host`-header clobber that + # can recurse a subrequest into itself) is in docs/4-implementation/networking/gateway.md's + # "Dialing another vhost by name" section — read it before touching + # this. + verifiedProxyTo = name: '' + proxy_ssl_verify on; + proxy_ssl_verify_depth 3; + proxy_ssl_trusted_certificate ${caBundle}; + proxy_ssl_name ${name}; + proxy_ssl_server_name on; + proxy_set_header Host ${name}; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Server $hostname; + ''; + + # `verifiedProxyTo`, plus the client certificate nginx presents to an + # upstream that demands one (`ssl_verify_client on`). Opt-in per call: the + # plain form sends none, which is what every existing caller wants. + verifiedProxyToWithClientCert = + name: + { certificate, key }: + verifiedProxyTo name + + '' + proxy_ssl_certificate ${certificate}; + proxy_ssl_certificate_key ${key}; + ''; + hstsDirectives = lib.concatStringsSep "; " ( [ "max-age=${toString cfg.hsts.maxAge}" ] ++ lib.optional cfg.hsts.includeSubDomains "includeSubDomains" @@ -82,26 +115,7 @@ in else vhostTls; - # Dial another service on this hive BY NAME over https, verified. - # nginx verifies nothing by default (`proxy_ssl_verify` is off), so a - # `proxy_pass https://…` without this is encrypted but unauthenticated - # — invisibly, it works and keeps working against any certificate at - # all. Full reasoning for every directive here (plus two real - # footguns — session-cache keying, and a `Host`-header clobber that - # can recurse a subrequest into itself) is in docs/4-implementation/networking/gateway.md's - # "Dialing another vhost by name" section — read it before touching - # this. - verifiedProxyTo = name: '' - proxy_ssl_verify on; - proxy_ssl_verify_depth 3; - proxy_ssl_trusted_certificate ${caBundle}; - proxy_ssl_name ${name}; - proxy_ssl_server_name on; - proxy_set_header Host ${name}; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Server $hostname; - ''; + inherit verifiedProxyTo verifiedProxyToWithClientCert; # Security headers added at the server scope on every vhost. # nginx's add_header inheritance rule: a location that defines its diff --git a/nix/host-modules/hive-tls.nix b/nix/host-modules/hive-tls.nix index bbbadb033..d840668d6 100644 --- a/nix/host-modules/hive-tls.nix +++ b/nix/host-modules/hive-tls.nix @@ -292,6 +292,13 @@ let BAO_CACERT = baoDeploy.serverCaFile; }; storeRetry = import ./lib/store-retry.nix { }; + + # The controller's TCP listener (./swarm-controller.nix): the gateway that + # fronts the controller presents a client leaf of its own to it. Only the + # host that runs the controller declares that vhost, so only it needs the leaf. + tcpCfg = hyperhiveCfg.swarm.controller.tcp; + swarmControllerCfg = hyperhiveCfg.deploy.swarm-controller; + gatewayClientLeaf = swarmControllerCfg.enable && tcpCfg.enable; in { # Host-side TLS for the self-signed gateway mode. @@ -953,6 +960,125 @@ in }; }; + # The gateway's client leaf for the controller's TCP listener, issued by + # the store's `swarm-controller-gateway-client` role: `clientAuth` only, + # one name, so the swarm-services leaf is not a substitute. Logs in with + # the services-issuer identity this host already holds, whose policy gains + # the one issue path where the listener is on. + # + # nginx names both files in `proxy_ssl_certificate*`, and refuses to start + # on a config naming a missing file, so a pair always exists once this + # unit has run: a throwaway self-signed one the listener refuses stands in + # until the store answers, and the controller's locations answer 502 in + # the meantime instead of the whole gateway staying down. + systemd.services.swarm-controller-gateway-client = lib.mkIf gatewayClientLeaf { + description = "Issue the gateway's client certificate for the swarm-controller's TCP listener"; + wantedBy = [ "multi-user.target" ]; + before = [ "nginx.service" ]; + after = [ + "container@${baoCfg.machine}.service" + "swarm-bao-pki.service" + "swarm-bao-services-issuer-policy.service" + ]; + path = servicesCertPath; + inherit (storeRetry) startLimitBurst startLimitIntervalSec; + serviceConfig = storeRetry.serviceConfig // { + Type = "oneshot"; + UMask = "0077"; + SyslogIdentifier = "swarm-controller-gateway-client"; + TimeoutStartSec = servicesCertTimeoutStartSec; + }; + environment = servicesCertEnvironment; + script = '' + set -euo pipefail + d=${lib.escapeShellArg cfg.stateDir} + cert=${lib.escapeShellArg swarmControllerCfg.tcp.gatewayClientCertFile} + key=${lib.escapeShellArg swarmControllerCfg.tcp.gatewayClientKeyFile} + root="$d/swarm-services-root.pem" + name=${lib.escapeShellArg (toString tcpCfg.clientName)} + install -d -m 0755 "$d" + + # Same readability as the gateway's own key: nginx's pre-start config + # test runs as the nginx user. + place() { # $1 = certificate, $2 = key, written in place of the pair + chgrp nginx "$2" + chmod 0640 "$2" + chmod 0644 "$1" + mv -f "$2" "$key" + mv -f "$1" "$cert" + } + + if [ ! -s "$cert" ] || [ ! -s "$key" ]; then + echo "no client certificate yet — placing a throwaway one so nginx can load its config" + openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \ + -days 1 -subj "/CN=placeholder" -keyout "$key.new" -out "$cert.new" 2>/dev/null + place "$cert.new" "$key.new" + fi + + # Missing, past half its window, naming something else, not chaining + # to the store's root, or not usable as a client certificate. The + # throwaway above fails the chain test, which is what replaces it. + reissue=0 + openssl x509 -in "$cert" -noout -checkend ${ + toString (baoDeploy.servicesPkiLeafTtlHours * 3600 / 2) + } >/dev/null 2>&1 || reissue=1 + openssl x509 -in "$cert" -noout -checkhost "$name" 2>/dev/null | grep -q ' does match ' || reissue=1 + { [ -s "$root" ] && openssl verify -CAfile "$root" -purpose sslclient "$cert" >/dev/null 2>&1; } || reissue=1 + if [ "$reissue" = 0 ]; then + echo "gateway client leaf valid for $name — leaving it alone" + exit 0 + fi + + ${requireServicesIssuer "ask the store for the gateway's client leaf"} + + err="$(mktemp)" + resp="$(mktemp "$d/controller-client.json.XXXXXX")" + trap 'rm -f "$err" "$resp"' EXIT + + if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then + echo "could not log in to the swarm secret store with this host's services-issuer certificate." >&2 + cat "$err" >&2 + exit 1 + fi + export BAO_TOKEN + + echo "requesting the gateway's client leaf for $name from the store" + if ! bao write -format=json \ + ${lib.escapeShellArg "${baoServicesPkiMount}/issue/${tcpCfg.clientPkiRole}"} \ + common_name="$name" > "$resp" 2>"$err"; then + echo "the store refused to issue the gateway's client certificate." >&2 + cat "$err" >&2 + exit 1 + fi + + jq -r '.data.private_key // empty' < "$resp" > "$key.new" + jq -r '.data.certificate // empty' < "$resp" > "$cert.new" + for f in "$key.new" "$cert.new"; do + if [ ! -s "$f" ]; then + echo "the store's response was missing a field: $f is empty" >&2 + exit 1 + fi + done + place "$cert.new" "$key.new" + + # `--no-block` for the reason `swarm-services-cert` gives: this unit + # is ordered before nginx, so a blocking reload would wait on us. + if systemctl is-active --quiet nginx.service; then + echo "gateway client leaf rotated — reloading nginx" + systemctl reload --no-block nginx.service + fi + ''; + }; + + systemd.timers.swarm-controller-gateway-client = lib.mkIf gatewayClientLeaf { + description = "Daily renewal check for the gateway's controller client certificate"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = "daily"; + Persistent = true; + }; + }; + # Daily rather than weekly, for the reason # ./swarm-nats.nix's `swarm-bao-nats-tls` timer is: this leaf needs the # store, which can be sealed or unreachable, so there is a retry per day diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 6f666f54a..8bcf93e39 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -650,6 +650,13 @@ let path "${sshAgentsMountPath}/config/ca" { capabilities = ["read"] } + '' + # The controller's TCP listener leaf: one issue path on a role of its own, + # present only where the listener is on. + + lib.optionalString tcpCfg.enable '' + path "${servicesPkiMountPath}/issue/${tcpCfg.serverPkiRole}" { + capabilities = ["update"] + } ''; # The identity authelia's mint creates its OIDC client secrets in the store @@ -979,6 +986,10 @@ let # and has to spell it the same way. servicesPkiMountPath = baoDeploy.servicesPkiMountPath; + # The controller's optional TCP listener: its two roles are written here, and + # its two principals are granted the matching issue paths below. + tcpCfg = hyperhiveCfg.swarm.controller.tcp; + # Subject of the root generated into that mount. A label for a human reading # a chain, not an identity anything authenticates against. Falls through so # a hive that has set neither a swarm name nor a domain still evaluates. @@ -1015,6 +1026,38 @@ let servicesPkiLeafTtl = "${toString servicesPkiLeafTtlHours}h"; servicesPkiLeafTtlSeconds = servicesPkiLeafTtlHours * 3600; + # The two roles, as script to splice after the services role's write. + # Empty where the listener is off, so the unit's script is the one it was + # before. Same narrowing as the services role, one name each: + # * `serverPkiRole` issues the listener's server leaf for `tcp.domain` + # alone, `serverAuth` only. + # * `clientPkiRole` issues the gateway's client leaf for `tcp.clientName` + # alone, `clientAuth` only. The opposite flag pair from the services + # role, which is why the services leaf cannot present itself to the + # listener. + tcpRoleWrite = role: name: flags: '' + bao write ${lib.escapeShellArg "${servicesPkiMountPath}/roles/${role}"} \ + allowed_domains=${lib.escapeShellArg (toString name)} \ + allow_bare_domains=true \ + allow_subdomains=false \ + allow_glob_domains=false \ + allow_localhost=false \ + allow_any_name=false \ + allow_ip_sans=false \ + enforce_hostnames=true \ + ${flags} \ + key_type=rsa \ + key_bits=4096 \ + ttl=${servicesPkiLeafTtl} \ + max_ttl=${servicesPkiLeafTtl} + ''; + tcpRoles = lib.optionalString tcpCfg.enable ( + "\n\n" + + tcpRoleWrite tcpCfg.serverPkiRole tcpCfg.domain "server_flag=true client_flag=false" + + "\n" + + tcpRoleWrite tcpCfg.clientPkiRole tcpCfg.clientName "server_flag=false client_flag=true" + ); + # The role every swarm-service certificate will be issued through, and the # whole narrowing of this mount. A role rather than a name-constrained # sub-CA: the engine enforces "these names and no others" at issue time, so @@ -1049,6 +1092,15 @@ let path "${servicesPkiMountPath}/issue/${servicesPkiRoleName}" { capabilities = ["update"] } + '' + # The gateway's client leaf for the controller's TCP listener. Every gateway + # host holds this identity, so every gateway host can obtain the leaf: what the + # controller trusts is "a gateway of this swarm". The role issues one name, + # with `clientAuth` only. + + lib.optionalString tcpCfg.enable '' + path "${servicesPkiMountPath}/issue/${tcpCfg.clientPkiRole}" { + capabilities = ["update"] + } ''; # The queue's principal: one `update` on its own role, the shape of the @@ -4057,7 +4109,7 @@ in key_type=rsa \ key_bits=4096 \ ttl=${servicesPkiLeafTtl} \ - max_ttl=${servicesPkiLeafTtl} + max_ttl=${servicesPkiLeafTtl}${tcpRoles} # `certificate=` is the CA, so this role trusts every leaf that # CA signed and `allowed_common_names` is the whole narrowing — diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix index 10faf96d7..11133bd98 100644 --- a/nix/host-modules/swarm-controller.nix +++ b/nix/host-modules/swarm-controller.nix @@ -7,6 +7,13 @@ # nginx is the only intended client, it runs on this same host and so # reaches the socket by path, and a socket that is never bound to an # address cannot be reached from off-host by mistake. +# +# `swarm.controller.tcp.enable` adds a TLS listener beside the socket, which the +# gateway then dials instead of the socket. The controller checks nobody per +# request, so the listener completes a handshake for one identity only: a +# client certificate the swarm's store issued for the gateway, with the +# `clientAuth` usage. ./swarm-ui.nix still declares the vhost on the host that +# runs this daemon, so the gateway stays on that host. { pkgs, lib, @@ -137,6 +144,16 @@ let swarmDomain = config.services.hyperhive.swarm.domain; matrixCfg = config.services.hyperhive.swarm.matrix; + # The optional TCP listener. `tcpCfg` is the swarm-wide half, evaluated + # identically on every host: the store's host writes its roles from it, the + # gateway's host issues its client leaf from it, and this host serves it. + # `tcpHost` is what only this host decides. + tcpCfg = cfg.tcp; + tcpHost = deployCfg.swarm-controller.tcp; + hiveTls = deployCfg.hive-controller.tls; + storeRetry = import ./lib/store-retry.nix { }; + tcpOn = deployCfg.swarm-controller.enable && tcpCfg.enable; + # Which swarm `swarmctl login` logs in to when given none: this host's own. # Read by `login` alone. `SWARMCTL_SWARM` would also move every socket verb # onto the gateway, where each needs a login. A default, so an operator's @@ -215,28 +232,29 @@ let }; # How the forge must address this controller to deliver a swarm-wide - # webhook. ./swarm-ui.nix declares the vhost and the `/webhook/forge/` - # location inside it on every host that runs this daemon, so the only gate - # left is a swarm domain to name it by. + # webhook: `deploy.swarm-controller.publicUrl`. ./swarm-ui.nix declares the + # vhost and the `/webhook/forge/` location inside it on every host that runs + # this daemon, so the only gate left is a URL to name it by. # # 🔑 The gate is the point, not a detail. The daemon registers hooks only # when this is set, and a hook whose `target_url` nothing answers is worse # than no hook at all — forgejo keeps the registration, marks every # delivery failed, and the hook still reads as configured. - # - # ⚠️ Correct ONLY while the vhost and this daemon share a host, and they do - # by construction: the UI's `/api/` location proxies - # `http://unix:`, a path that resolves nowhere else, and this - # daemon binds no TCP address at all (see the header). - # - # ⇒ The day this daemon grows a TCP listener so another host can front it, - # THIS LINE SILENTLY STOPS BEING RIGHT: hooks get registered against a vhost - # that may no longer be the one fronting the daemon, and nothing errors. That is - # the moment to add an explicit `publicUrl` option — not before, because - # until then there is exactly one derivable answer and an option would only - # be a second place to get it wrong. - webhookEnv = lib.optionalAttrs (swarmDomain != null) { - SWARM_CONTROLLER_PUBLIC_URL = "https://${swarmDomain}"; + webhookEnv = lib.optionalAttrs (deployCfg.swarm-controller.publicUrl != null) { + SWARM_CONTROLLER_PUBLIC_URL = deployCfg.swarm-controller.publicUrl; + }; + + # What the daemon needs to open its TCP listener (`swarm-controller/src/ + # tcp_tls.rs`). All five or none: the daemon refuses a partial set. The + # client CA is the store's services root, the one the gateway's client leaf + # chains to, from the file `hive-tls-ca` already keeps on every self-signed + # gateway host. + tcpEnv = lib.optionalAttrs tcpOn { + SWARM_CONTROLLER_TCP_LISTEN = "${tcpHost.listenAddress}:${toString tcpCfg.port}"; + SWARM_CONTROLLER_TCP_CERT = "${tcpHost.tlsDir}/cert.pem"; + SWARM_CONTROLLER_TCP_KEY = "${tcpHost.tlsDir}/key.pem"; + SWARM_CONTROLLER_TCP_CLIENT_CA = "${hiveTls.stateDir}/swarm-services-root.pem"; + SWARM_CONTROLLER_TCP_CLIENT_NAME = tcpCfg.clientName; }; # Not a secret to deliver — `swarm-authelia-bridge`'s own bearer check @@ -396,6 +414,84 @@ in ''; }; + # Swarm tier, not `deploy.*`: the store's host writes the two roles below, + # the host fronting the controller issues the gateway's client leaf, and the + # controller's host serves — three hosts that must agree on one set of + # names, so each reads them from the option every host evaluates alike. + tcp = { + enable = lib.mkEnableOption '' + the controller's TCP listener, TLS with a required client certificate. + Off: the controller serves its unix socket only, which is what the + gateway on the same host reaches it through. + + On: the gateway's controller locations dial {option}`domain` over TLS + and present a client leaf, and the controller accepts that one + identity. The unix socket keeps working beside it. The listener + reaches only the gateway: the controller trusts its transport and + checks nobody per request, so a certificate the swarm root signed for + anything else is refused at the handshake. + + Set it identically on every host of the swarm. The store's host + writes the two roles from it, the controller's host issues its server + leaf, and the host fronting the controller issues the gateway's client + leaf, each through an identity it already holds. + ''; + + domain = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "controller.example.org"; + description = '' + The name the gateway dials the controller by. It becomes the server + certificate's name (the store's `swarm-controller-tls` role issues + for this name alone) and what nginx verifies the controller against. + It must resolve, from the gateway's host, to an address the + controller listens on. Required when {option}`enable` is set. + ''; + }; + + port = lib.mkOption { + type = lib.types.port; + default = 9443; + description = "TCP port the controller listens on and the gateway dials."; + }; + + clientName = lib.mkOption { + type = lib.types.nullOr lib.types.str; + readOnly = true; + default = if swarmDomain == null then null else "gateway-client.${swarmDomain}"; + defaultText = lib.literalExpression ''"gateway-client.''${services.hyperhive.swarm.domain}"''; + description = '' + The one name a client certificate must carry for the controller to + accept it. The store's `swarm-controller-gateway-client` role issues for this + name alone, with the `clientAuth` usage and not `serverAuth`, so the + swarm-services leaf the gateway serves browsers with does not + qualify. + ''; + }; + + serverPkiRole = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = "swarm-controller-tls"; + description = '' + The store's `pki` role that issues the controller's server leaf. + Starts with `swarm-`: the granter that writes it may write roles + under that prefix only. + ''; + }; + + clientPkiRole = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = "swarm-controller-gateway-client"; + description = '' + The store's `pki` role that issues the gateway's client leaf. Starts + with `swarm-`, as {option}`serverPkiRole` does. + ''; + }; + }; + }; # What stays above is what the controller IS to the swarm: @@ -610,6 +706,86 @@ in ''; }; + publicUrl = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = if swarmDomain == null then null else "https://${swarmDomain}"; + defaultText = lib.literalExpression ''"https://''${services.hyperhive.swarm.domain}"''; + example = "https://swarm.example.org"; + description = '' + The URL the forge reaches this controller at to deliver webhooks, + which the daemon registers the swarm's hooks against + (`SWARM_CONTROLLER_PUBLIC_URL`). It names the vhost that fronts the + controller, not the controller: the apex gateway on this host answers + it whether the controller is behind its socket or its TCP listener. + Set it when another URL fronts the controller. + + `null` registers no hooks. + ''; + }; + + tcp = { + listenAddress = lib.mkOption { + type = lib.types.str; + default = "0.0.0.0"; + example = "10.0.0.5"; + description = '' + Address the controller binds when + {option}`services.hyperhive.swarm.controller.tcp.enable` is set. An + IPv6 address is written in brackets. Binding a port opens nothing: + {option}`openFirewall` decides who can reach it, and the TLS + handshake decides who may talk once they do. + ''; + }; + + openFirewall = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Open {option}`services.hyperhive.swarm.controller.tcp.port` in this + host's firewall. Off by default: whether the controller is reachable + from off-host is the operator's decision, and a gateway on this same + host needs nothing opened. Turn it on when the gateway that fronts + the controller runs elsewhere. + ''; + }; + + tlsDir = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = "/var/lib/swarm-controller-tls"; + description = '' + Where `swarm-controller-tls.service` writes the server leaf and key + the listener serves: `cert.pem` and `key.pem`, the key readable by + the controller's group alone. The daemon re-reads both when they + change, so a rotation needs no restart. + ''; + }; + + gatewayClientCertFile = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = "${hiveTls.stateDir}/controller-client.pem"; + defaultText = lib.literalExpression ''"''${deploy.hive-controller.tls.stateDir}/controller-client.pem"''; + description = '' + The client certificate the gateway on this host presents to the + controller's listener. `swarm-controller-gateway-client.service` + issues it from the store, and the gateway's controller locations + name it in `proxy_ssl_certificate`. + ''; + }; + + gatewayClientKeyFile = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = "${hiveTls.stateDir}/controller-client-key.pem"; + defaultText = lib.literalExpression ''"''${deploy.hive-controller.tls.stateDir}/controller-client-key.pem"''; + description = '' + Private key for {option}`gatewayClientCertFile`, mode `0640` and group + `nginx`, the way the gateway's own key is readable. + ''; + }; + }; + swarmctlEnvironment = lib.mkOption { type = lib.types.attrsOf lib.types.str; default = { }; @@ -751,12 +927,35 @@ in in both options. ''; } + { + assertion = !tcpCfg.enable || (tcpCfg.domain != null && tcpCfg.clientName != null); + message = '' + services.hyperhive.swarm.controller.tcp.enable needs + services.hyperhive.swarm.controller.tcp.domain, the name the gateway + dials the controller by, and services.hyperhive.swarm.domain, which + the gateway's client identity is named under. Set both, or leave the + TCP listener off. + ''; + } + { + assertion = !tcpCfg.enable || caTrust.useSelfSigned; + message = '' + services.hyperhive.swarm.controller.tcp.enable needs the self-signed + gateway mode: the controller trusts the secret store's services root + for client certificates, and only a self-signed gateway host keeps + that root on disk. Leave the TCP listener off where the gateway + serves an operator certificate or ACME. + ''; + } ]; systemd.services.swarm-controller = { description = "hyperhive swarm-level controller daemon"; wantedBy = [ "multi-user.target" ]; - after = [ "network.target" ]; + # The listener's certificate files must exist when the daemon opens it. A + # start that finds them missing fails and `Restart=` tries again. + after = [ "network.target" ] ++ lib.optional tcpOn "swarm-controller-tls.service"; + wants = lib.optional tcpOn "swarm-controller-tls.service"; serviceConfig = { ExecStart = "${deployCfg.swarm-controller.package}/bin/swarm-controller"; @@ -901,7 +1100,113 @@ in // swarmNameEnv // matrixHomeserverEnv // baoEnv - // metaEnv; + // metaEnv + // tcpEnv; + }; + + networking.firewall.allowedTCPPorts = lib.mkIf (tcpOn && tcpHost.openFirewall) [ tcpCfg.port ]; + + # The listener's server leaf, from the store's `${tcpCfg.serverPkiRole}` + # role under the controller's own store identity — the one it already logs + # in with, whose policy gains this one issue path when the listener is on. + # Same shape as the queue's leaf unit (./swarm-nats.nix, + # `swarm-bao-nats-tls`): the daemon re-reads the files, so rotating needs + # no restart, which for this daemon would drop its job graph. + systemd.services.swarm-controller-tls = lib.mkIf tcpOn { + description = "Issue the swarm-controller's TCP listener certificate from the secret store's PKI"; + wantedBy = [ "multi-user.target" ]; + after = [ + "container@${baoCfg.machine}.service" + "swarm-bao-pki.service" + "swarm-bao-controller-policy.service" + ]; + path = [ + deployCfg.bao.package + pkgs.jq + pkgs.openssl + pkgs.coreutils + pkgs.gnugrep + ]; + # The store is sealed or unreachable for as long as it takes an operator + # to unseal it; see ./lib/store-retry.nix. + inherit (storeRetry) startLimitBurst startLimitIntervalSec; + serviceConfig = storeRetry.serviceConfig // { + Type = "oneshot"; + UMask = "0077"; + SyslogIdentifier = "swarm-controller-tls"; + TimeoutStartSec = "5min"; + }; + environment = { + BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; + BAO_CLIENT_CERT = deployCfg.swarm-controller.baoClientCertFile; + BAO_CLIENT_KEY = deployCfg.swarm-controller.baoClientKeyFile; + } + // lib.optionalAttrs (deployCfg.bao.serverCaFile != null) { + BAO_CACERT = deployCfg.bao.serverCaFile; + }; + script = '' + set -euo pipefail + d=${lib.escapeShellArg tcpHost.tlsDir} + name=${lib.escapeShellArg tcpCfg.domain} + install -d -m 0750 -g swarm-controller "$d" + + # Missing, past half its window, or naming something other than the + # configured domain. + reissue=0 + { [ -s "$d/cert.pem" ] && [ -s "$d/key.pem" ]; } || reissue=1 + openssl x509 -in "$d/cert.pem" -noout -checkend ${ + toString (deployCfg.bao.servicesPkiLeafTtlHours * 3600 / 2) + } >/dev/null 2>&1 || reissue=1 + openssl x509 -in "$d/cert.pem" -noout -checkhost "$name" 2>/dev/null | grep -q ' does match ' || reissue=1 + if [ "$reissue" = 0 ]; then + echo "controller listener leaf valid for $name — leaving it alone" + exit 0 + fi + + err="$(mktemp)" + resp="$(mktemp "$d/issue.json.XXXXXX")" + trap 'rm -f "$err" "$resp"' EXIT + + if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then + echo "could not log in to the swarm secret store with the controller's certificate." >&2 + cat "$err" >&2 + exit 1 + fi + export BAO_TOKEN + + echo "requesting the controller's listener leaf for $name from the store" + if ! bao write -format=json \ + ${lib.escapeShellArg "${deployCfg.bao.servicesPkiMountPath}/issue/${tcpCfg.serverPkiRole}"} \ + common_name="$name" > "$resp" 2>"$err"; then + echo "the store refused to issue the controller's listener certificate." >&2 + cat "$err" >&2 + exit 1 + fi + + jq -r '.data.private_key // empty' < "$resp" > "$d/key.pem.new" + jq -r '.data.certificate // empty' < "$resp" > "$d/cert.pem.new" + for f in "$d/key.pem.new" "$d/cert.pem.new"; do + if [ ! -s "$f" ]; then + echo "the store's response was missing a field: $f is empty" >&2 + exit 1 + fi + done + chgrp swarm-controller "$d/key.pem.new" "$d/cert.pem.new" + chmod 0640 "$d/key.pem.new" "$d/cert.pem.new" + # Key first: the daemon pairs whatever it reads, and a new key beside + # the old certificate is a pair it refuses and retries. + mv -f "$d/key.pem.new" "$d/key.pem" + mv -f "$d/cert.pem.new" "$d/cert.pem" + ''; + }; + + systemd.timers.swarm-controller-tls = lib.mkIf tcpOn { + description = "Daily renewal check for the swarm-controller's TCP listener certificate"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = "daily"; + Persistent = true; + }; }; }; } diff --git a/nix/host-modules/swarm-ui.nix b/nix/host-modules/swarm-ui.nix index 825bfe414..8c1f45118 100644 --- a/nix/host-modules/swarm-ui.nix +++ b/nix/host-modules/swarm-ui.nix @@ -106,12 +106,46 @@ let matrixCfg = swarmCfg.matrix; serverName = if matrixCfg.serverName != null then matrixCfg.serverName else apex; servesMatrixDiscovery = matrixCfg.gatewayHost != null && serverName == apex; + + # How the gateway reaches the controller. Its unix socket by default. With + # the controller's TCP listener on, over TLS by the listener's name, with + # the gateway's client leaf — the listener refuses a handshake without it + # (./swarm-controller.nix). `controllerDial` is empty on the socket, so the + # two locations below render exactly what they rendered before the + # listener existed. + # `toString` below so an unset `domain` reaches the assertion that names it + # instead of failing evaluation first. + controllerTcp = swarmCfg.controller.tcp; + controllerUpstream = + if controllerTcp.enable then + "https://${toString controllerTcp.domain}:${toString controllerTcp.port}" + else + "http://unix:${deployCfg.swarm-controller.socketPath}:"; + # `proxy_http_version` and `Connection` because `recommendedProxySettings`, + # which sets them, is off on a location that pins its own `Host`: without + # them nginx speaks HTTP/1.0 to the upstream, which ends the SSE streams' + # keep-alive. + controllerDial = lib.optionalString controllerTcp.enable '' + ${gatewayCfg.lib.verifiedProxyToWithClientCert (toString controllerTcp.domain) { + certificate = deployCfg.swarm-controller.tcp.gatewayClientCertFile; + key = deployCfg.swarm-controller.tcp.gatewayClientKeyFile; + }} + proxy_http_version 1.1; + proxy_set_header Connection ""; + ''; + controllerDialAttrs = lib.optionalAttrs controllerTcp.enable { + # nixpkgs appends its own `Host $host` after `extraConfig`, which would + # override the pinned one — see the comment on verifiedProxyTo in + # hive-gateway/vhost-lib.nix. + recommendedProxySettings = false; + }; in { # No `enable` of its own: the UI is on wherever # `services.hyperhive.deploy.swarm-controller.enable` is, because it reaches - # that daemon over its unix socket. There is no `domain` option: the UI - # answers on the swarm apex, which the swarm already declares once. + # that daemon over its unix socket, or over its TCP listener when that is on. + # There is no `domain` option: the UI answers on the swarm apex, which the + # swarm already declares once. # ⚠️ Neither `services.hyperhive.swarm.ui` nor `deploy.swarm-ui` is a # namespace for anything but the old spellings: the UI's `package` lives @@ -295,8 +329,8 @@ in # shape as the per-hive dashboard's own `/api/` proxy) so the # path swarm-controller registered a route at is the path # nginx forwards, no prefix-stripping to keep in sync by hand. - "/api/" = { - proxyPass = "http://unix:${deployCfg.swarm-controller.socketPath}:"; + "/api/" = controllerDialAttrs // { + proxyPass = controllerUpstream; # `Remote-User` from authelia's answer, set here so it replaces # any the client sent: swarm-controller takes it as the sender of # the text the operator types to an agent (`agent_input.rs`). @@ -313,7 +347,8 @@ in # must not sit in nginx's buffer. The controller's 15s # keep-alive keeps them under nginx's 60s default read timeout. proxy_buffering off; - ''; + '' + + controllerDial; }; # A refused request gets the login page only when it asks for a page. # The swarm UI calls `/api/` with `fetch` and `EventSource`, and a @@ -356,9 +391,14 @@ in # carve-out is exactly as wide as the endpoint that justifies it: # a future `/webhook/` does not inherit the # bypass by living under a shared prefix. - "/webhook/forge/" = { - proxyPass = "http://unix:${deployCfg.swarm-controller.socketPath}:"; - }; + "/webhook/forge/" = + controllerDialAttrs + // { + proxyPass = controllerUpstream; + } + // lib.optionalAttrs controllerTcp.enable { + extraConfig = controllerDial; + }; # Swagger UI: same "nginx hosts the themed dist straight from # the store, only /api/openapi.json is dynamic" shape as the # hive dashboard's `swaggerUiLocations` (./hive-c0re/vhosts.nix). diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index 894ae8853..74da1cc96 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -41,6 +41,26 @@ let deploy.hive-controller.hyperhiveRev = lib.fixedWidthString 40 "b" ""; }; + # `allLocal` with the controller's TCP listener on, the one fixture that has + # the store, the controller and the gateway together: the two roles, the two + # issue grants and the vhost all render from this one option. The firewall is + # opened so the lookup that finds the port is shown to find it. Off is + # `allLocal` itself. + allLocalTcp = hiveWithController { + deploy.singleHostSwarm = true; + swarm.controller.tcp = { + enable = true; + domain = "controller.t.local"; + }; + deploy.swarm-controller.tcp.openFirewall = true; + }; + + # The listener asked for with nothing to name it by. + tcpNoDomain = hive { + deploy.swarm-controller.enable = true; + swarm.controller.tcp.enable = true; + }; + bare = shared.bareWithController; # `lib.hasInfix` matches by regex, which refuses a needle that refers to a @@ -3016,6 +3036,149 @@ let mentions "privateListenAddresses holds" && !(mentions "while services.hyperhive.gateway.privateListenAddresses is empty"); } + { + # The off arm of the controller's TCP listener: a host that never set it + # renders what it rendered before it existed. The gateway still reaches + # the controller by socket, nothing TCP-shaped is rendered, and the + # store writes neither role nor grant. The `swarm-services-cert` unit is + # the presence control for the unit lookup, and the socket in + # `proxyPass` for the location lookup. + name = "a swarm without the controller's TCP listener renders the socket-only controller"; + ok = + let + m = allLocal; + l = m.services.nginx.virtualHosts."t.local".locations; + socket = "http://unix:${m.services.hyperhive.deploy.swarm-controller.socketPath}:"; + env = m.systemd.services.swarm-controller.environment; + policies = m.systemd.services.swarm-bao-controller-policy.script; + issuer = m.systemd.services.swarm-bao-services-issuer-policy.script; + in + l."/api/".proxyPass == socket + && l."/webhook/forge/".proxyPass == socket + && !(lib.hasInfix "proxy_ssl_certificate" l."/api/".extraConfig) + && !(lib.hasInfix "proxy_ssl" l."/webhook/forge/".extraConfig) + && !(m.systemd.services ? swarm-controller-tls) + && !(m.systemd.services ? swarm-controller-gateway-client) + && !(m.systemd.timers ? swarm-controller-tls) + && !(m.systemd.timers ? swarm-controller-gateway-client) + && m.systemd.services ? swarm-services-cert + && !(lib.any (lib.hasPrefix "SWARM_CONTROLLER_TCP_") (lib.attrNames env)) + && !(lib.elem 9443 m.networking.firewall.allowedTCPPorts) + && !(lib.hasInfix "swarm-controller-tls" policies) + && !(lib.hasInfix "swarm-controller-gateway-client" policies) + && !(lib.hasInfix "swarm-controller-gateway-client" issuer) + && lib.hasInfix "issue/swarm-services" issuer + && env.SWARM_CONTROLLER_PUBLIC_URL == "https://t.local" + && m.services.hyperhive.deploy.swarm-controller.tcp.openFirewall == false + && allLocalTcp.services.hyperhive.deploy.swarm-controller.tcp.openFirewall == true; + } + { + # The on arm. The gateway dials the listener by name over TLS, verified, + # with the client leaf, and keeps the operator gate on `/api/` and the + # webhook's absence of it; the controller is handed all five listener + # variables, naming the services root as the client CA and the gateway's + # one name as the client; the firewall port is open because the fixture + # asked. + name = "the controller's TCP listener is dialled over verified TLS with the gateway's client leaf"; + ok = + let + m = allLocalTcp; + l = m.services.nginx.virtualHosts."t.local".locations; + upstream = "https://controller.t.local:9443"; + tls = m.services.hyperhive.deploy.hive-controller.tls.stateDir; + env = m.systemd.services.swarm-controller.environment; + dialled = + loc: + lib.all (s: lib.hasInfix s loc.extraConfig) [ + "proxy_ssl_verify on;" + "proxy_ssl_name controller.t.local;" + "proxy_ssl_certificate ${tls}/controller-client.pem;" + "proxy_ssl_certificate_key ${tls}/controller-client-key.pem;" + "proxy_set_header Host controller.t.local;" + ]; + in + l."/api/".proxyPass == upstream + && l."/webhook/forge/".proxyPass == upstream + && dialled l."/api/" + && dialled l."/webhook/forge/" + && l."/api/".recommendedProxySettings == false + && l."/webhook/forge/".recommendedProxySettings == false + && lib.hasInfix "auth_request /__hive_authelia;" l."/api/".extraConfig + && !(lib.hasInfix "auth_request" l."/webhook/forge/".extraConfig) + && env.SWARM_CONTROLLER_TCP_LISTEN == "0.0.0.0:9443" + && env.SWARM_CONTROLLER_TCP_CERT == "/var/lib/swarm-controller-tls/cert.pem" + && env.SWARM_CONTROLLER_TCP_KEY == "/var/lib/swarm-controller-tls/key.pem" + && env.SWARM_CONTROLLER_TCP_CLIENT_CA == "${tls}/swarm-services-root.pem" + && env.SWARM_CONTROLLER_TCP_CLIENT_NAME == "gateway-client.t.local" + && lib.elem 9443 m.networking.firewall.allowedTCPPorts + && env.SWARM_CONTROLLER_PUBLIC_URL == "https://t.local" + && m.systemd.services.swarm-controller-tls.wantedBy == [ "multi-user.target" ] + && lib.elem "swarm-controller-tls.service" m.systemd.services.swarm-controller.after; + } + { + # The two roles carry opposite flag pairs, which is the whole reason the + # swarm-services leaf cannot present itself to the listener: the client + # role issues `clientAuth` only and the server role `serverAuth` only. + # Each is read out of the one `bao write` that names it, so swapped flags + # fail here and a role that is missing fails on the first clause. The + # grants are split the same way: the controller may issue its server + # leaf and the gateway-side identity its client leaf, and neither may + # issue the other's. + name = "the store writes a server-only and a client-only role and grants each to one principal"; + ok = + let + m = allLocalTcp; + script = m.systemd.services.swarm-bao-controller-policy.script; + write = + role: lib.findFirst (c: lib.hasInfix "/roles/${role} " c) "" (lib.splitString "bao write" script); + server = write "swarm-controller-tls"; + client = write "swarm-controller-gateway-client"; + issuer = m.systemd.services.swarm-bao-services-issuer-policy.script; + in + server != "" + && client != "" + && lib.hasInfix "allowed_domains=controller.t.local" server + && lib.hasInfix "server_flag=true" server + && lib.hasInfix "client_flag=false" server + && lib.hasInfix "allowed_domains=gateway-client.t.local" client + && lib.hasInfix "server_flag=false" client + && lib.hasInfix "client_flag=true" client + && lib.hasInfix "allow_any_name=false" client + && lib.hasInfix "issue/swarm-controller-tls" script + && !(lib.hasInfix "issue/swarm-controller-gateway-client" script) + && lib.hasInfix "issue/swarm-controller-gateway-client" issuer + && !(lib.hasInfix "issue/swarm-controller-tls" issuer) + && lib.hasInfix "issue/swarm-services" issuer; + } + { + # The gateway's client leaf is checked for the `clientAuth` purpose + # before it is trusted as current, so a leaf issued under the wrong role + # is replaced rather than kept. The throwaway pair that stands in until + # the store answers is what keeps nginx's config loadable. + name = "the gateway's client leaf is issued with the services-issuer identity and placed for nginx"; + ok = + let + u = allLocalTcp.systemd.services.swarm-controller-gateway-client; + in + lib.hasInfix "-purpose sslclient" u.script + && lib.hasInfix "issue/swarm-controller-gateway-client" u.script + && lib.hasInfix "chgrp nginx" u.script + && lib.hasInfix "placeholder" u.script + && lib.elem "nginx.service" u.before + && u.environment ? BAO_ADDR + && allLocalTcp.systemd.timers ? swarm-controller-gateway-client; + } + { + name = "the controller's TCP listener is refused at eval with no name to dial it by"; + ok = + let + fired = lib.filter (a: !a.assertion) tcpNoDomain.assertions; + in + lib.any (a: lib.hasInfix "swarm.controller.tcp.domain" a.message) fired + && !(lib.any (a: lib.hasInfix "swarm.controller.tcp.domain" a.message) ( + lib.filter (a: !a.assertion) allLocalTcp.assertions + )); + } ] # Starting a unit that stays `active (exited)` is a no-op, so a timer aimed # at one fires and fetches nothing. diff --git a/swarm-controller/src/tcp_tls.rs b/swarm-controller/src/tcp_tls.rs index b07827aa0..217fb7e54 100644 --- a/swarm-controller/src/tcp_tls.rs +++ b/swarm-controller/src/tcp_tls.rs @@ -175,7 +175,10 @@ fn read_pair(cert: &Path, key: &Path, provider: &CryptoProvider) -> Result