hive-tls: a store-only host without a hive starts its gateway with the store sealed
A gateway host without a hive whose only swarm name is the store's UI no longer waits for swarm-services-cert: its nginx carries the stream passthrough to the store and must come up while the store is sealed. Until the services leaf lands, hive-gateway-self-signed-cert installs a throwaway self-signed pair on the default and swarm-services paths there; on a host without a hive it never references the hive leaf. A host without a hive also no longer mints the hive-leaf identity or defaults the options pointing at it, and does not render the hive-domain vhost, whose upstream is hive-c0re. Refs #4588
This commit is contained in:
atlas
mara
parent
75a7dccee6
commit
60268b70b0
9 changed files with 168 additions and 82 deletions
|
|
@ -262,13 +262,16 @@ in
|
|||
[ -s ${pkiDir}/nats-auth.pem ] || ${signLeaf} ${pkiDir} nats-auth \
|
||||
${lib.escapeShellArg deployCfg.bao.natsAuthCommonName} "" clientAuth
|
||||
|
||||
# This hive's gateway-leaf identity, which opens
|
||||
# `pki/issue/<hiveLeafPkiRolePrefix>-<hive>` and nothing else. Its
|
||||
# subject carries this hive's name, like the matrix-token and
|
||||
# queue-agent leaves above, because ./swarm-bao.nix writes one role per
|
||||
# hive. Hives on other hosts are issued theirs out of band.
|
||||
[ -s ${pkiDir}/hive-leaf.pem ] || ${signLeaf} ${pkiDir} hive-leaf \
|
||||
${lib.escapeShellArg "${deployCfg.bao.hiveLeafCommonNamePrefix}-${clientCn}"} "" clientAuth
|
||||
${lib.optionalString deployCfg.hive-controller.enable ''
|
||||
# This hive's gateway-leaf identity, which opens
|
||||
# `pki/issue/<hiveLeafPkiRolePrefix>-<hive>` and nothing else. Its
|
||||
# subject carries this hive's name, like the matrix-token and
|
||||
# queue-agent leaves above, because ./swarm-bao.nix writes one role
|
||||
# per hive. Only where a hive runs; hives on other hosts are issued
|
||||
# theirs out of band.
|
||||
[ -s ${pkiDir}/hive-leaf.pem ] || ${signLeaf} ${pkiDir} hive-leaf \
|
||||
${lib.escapeShellArg "${deployCfg.bao.hiveLeafCommonNamePrefix}-${clientCn}"} "" clientAuth
|
||||
''}
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@
|
|||
# ⚠️ The minting is NOT here. ./glue-bao-tls.nix holds the CA and signs the
|
||||
# leaf, for the reason ./glue-services-issuer-bao-identity.nix gives.
|
||||
#
|
||||
# Gated on the minted PKI existing, as that file is.
|
||||
# Gated on the minted PKI existing, as that file is, and on a hive running
|
||||
# here: a host without one has no hive leaf to ask for.
|
||||
#
|
||||
# Everything is `mkDefault`. An operator naming their own paths wins.
|
||||
{
|
||||
|
|
@ -25,7 +26,7 @@ let
|
|||
pkiDir = if haveMintedPki then builtins.dirOf baoDeploy.clientCertFile else null;
|
||||
in
|
||||
{
|
||||
config = lib.mkIf haveMintedPki {
|
||||
config = lib.mkIf (haveMintedPki && config.services.hyperhive.deploy.hive-controller.enable) {
|
||||
services.hyperhive.deploy.hive-controller.tls = {
|
||||
baoHiveIdentityCertFile = lib.mkDefault "${pkiDir}/hive-leaf.pem";
|
||||
baoHiveIdentityKeyFile = lib.mkDefault "${pkiDir}/hive-leaf-key.pem";
|
||||
|
|
|
|||
|
|
@ -113,6 +113,7 @@ let
|
|||
matrixCfg
|
||||
matrixDeployCfg
|
||||
hyperhiveDomain
|
||||
runsHive
|
||||
dashboardDist
|
||||
swaggerUiTheme
|
||||
;
|
||||
|
|
@ -276,7 +277,8 @@ in
|
|||
# Pin the journal identity (else it's the `script` store-path wrapper).
|
||||
SyslogIdentifier = "hive-gateway-self-signed-cert";
|
||||
};
|
||||
path = [ pkgs.coreutils ];
|
||||
# openssl for the throwaway pair a host without a hive starts on.
|
||||
path = [ pkgs.coreutils ] ++ lib.optional (!runsHive) pkgs.openssl;
|
||||
script = ''
|
||||
set -eu
|
||||
mkdir -p ${tlsDir}
|
||||
|
|
@ -301,45 +303,62 @@ in
|
|||
if runsHive then
|
||||
''
|
||||
install -m 0644 ${tlsStateDir}/gateway.pem ${tlsCert}
|
||||
install -m 0640 -g nginx ${tlsStateDir}/gateway-key.pem ${tlsKey}''
|
||||
install -m 0640 -g nginx ${tlsStateDir}/gateway-key.pem ${tlsKey}
|
||||
|
||||
# The swarm-services leaf, when this host issues one. It is
|
||||
# a separate pair rather than more SANs on the one above
|
||||
# because the hive leaf's role allows only its own hive's
|
||||
# domain, and the service names are siblings of it.
|
||||
#
|
||||
# Absent is a normal state, not a failure: a host fronting
|
||||
# no swarm name has none, and issuance can also fail on a
|
||||
# host that wants one.
|
||||
#
|
||||
# ⚠️ When it is absent the HIVE leaf goes to this path
|
||||
# anyway, and that fallback is load-bearing rather than
|
||||
# tidy. nginx refuses to load a config naming a cert file
|
||||
# that does not exist — `cannot load certificate … no such
|
||||
# file` fails the pre-start test, so the vhost does not
|
||||
# degrade, the ENTIRE gateway dies and takes the forge, the
|
||||
# dashboard and matrix with it. Serving the hive leaf on a
|
||||
# swarm-service name is a name mismatch: browsers warn,
|
||||
# strict clients refuse, everything else keeps working, and
|
||||
# the operator gets a bad cert instead of no hive.
|
||||
#
|
||||
# Measured, not theorised: this exact path took pr1ma's
|
||||
# gateway down when the services sub-CA failed to issue.
|
||||
if [ -s ${tlsStateDir}/swarm-services.pem ]; then
|
||||
install -m 0644 ${tlsStateDir}/swarm-services.pem ${svcCert}
|
||||
install -m 0640 -g nginx ${tlsStateDir}/swarm-services-key.pem ${svcKey}
|
||||
else
|
||||
echo "no swarm-services leaf — serving the hive leaf on those names (mismatch, not an outage)" >&2
|
||||
install -m 0644 ${tlsStateDir}/gateway.pem ${svcCert}
|
||||
install -m 0640 -g nginx ${tlsStateDir}/gateway-key.pem ${svcKey}
|
||||
fi''
|
||||
else
|
||||
''
|
||||
# No hive here, so no hive leaf: the catch-all and any other
|
||||
# vhost on the hive cert path serve the swarm-services leaf.
|
||||
install -m 0644 ${tlsStateDir}/swarm-services.pem ${tlsCert}
|
||||
install -m 0640 -g nginx ${tlsStateDir}/swarm-services-key.pem ${tlsKey}''
|
||||
# No hive here, so no hive leaf: every vhost, the catch-all
|
||||
# included, serves the swarm-services leaf. Until it lands a
|
||||
# throwaway self-signed pair stands in, because nginx refuses
|
||||
# a config naming a missing cert file. A store-only host does
|
||||
# not wait for the leaf, so its stream passthrough comes up with
|
||||
# the store sealed. `swarm-services-cert` re-runs this unit
|
||||
# once the leaf lands.
|
||||
if [ -s ${tlsStateDir}/swarm-services.pem ] && [ -s ${tlsStateDir}/swarm-services-key.pem ]; then
|
||||
cert=${tlsStateDir}/swarm-services.pem
|
||||
key=${tlsStateDir}/swarm-services-key.pem
|
||||
else
|
||||
echo "no swarm-services leaf yet — serving a throwaway self-signed certificate until it lands" >&2
|
||||
cert=${tlsDir}/fallback.pem
|
||||
key=${tlsDir}/fallback-key.pem
|
||||
(umask 077; openssl req -x509 -newkey rsa:2048 -nodes -sha256 -days 30 \
|
||||
-subj "/CN=hyperhive-gateway-fallback" -keyout "$key" -out "$cert")
|
||||
fi
|
||||
install -m 0644 "$cert" ${tlsCert}
|
||||
install -m 0640 -g nginx "$key" ${tlsKey}
|
||||
install -m 0644 "$cert" ${svcCert}
|
||||
install -m 0640 -g nginx "$key" ${svcKey}''
|
||||
}
|
||||
|
||||
# The swarm-services leaf, when this host issues one. It is
|
||||
# a separate pair rather than more SANs on the one above
|
||||
# because the hive leaf's role allows only its own hive's
|
||||
# domain, and the service names are siblings of it.
|
||||
#
|
||||
# Absent is a normal state, not a failure: a host fronting
|
||||
# no swarm name has none, and issuance can also fail on a
|
||||
# host that wants one.
|
||||
#
|
||||
# ⚠️ When it is absent the HIVE leaf goes to this path
|
||||
# anyway, and that fallback is load-bearing rather than
|
||||
# tidy. nginx refuses to load a config naming a cert file
|
||||
# that does not exist — `cannot load certificate … no such
|
||||
# file` fails the pre-start test, so the vhost does not
|
||||
# degrade, the ENTIRE gateway dies and takes the forge, the
|
||||
# dashboard and matrix with it. Serving the hive leaf on a
|
||||
# swarm-service name is a name mismatch: browsers warn,
|
||||
# strict clients refuse, everything else keeps working, and
|
||||
# the operator gets a bad cert instead of no hive.
|
||||
#
|
||||
# Measured, not theorised: this exact path took pr1ma's
|
||||
# gateway down when the services sub-CA failed to issue.
|
||||
if [ -s ${config.services.hyperhive.deploy.hive-controller.tls.stateDir}/swarm-services.pem ]; then
|
||||
install -m 0644 ${config.services.hyperhive.deploy.hive-controller.tls.stateDir}/swarm-services.pem ${svcCert}
|
||||
install -m 0640 -g nginx ${config.services.hyperhive.deploy.hive-controller.tls.stateDir}/swarm-services-key.pem ${svcKey}
|
||||
else
|
||||
echo "no swarm-services leaf — serving the hive leaf on those names (mismatch, not an outage)" >&2
|
||||
install -m 0644 ${config.services.hyperhive.deploy.hive-controller.tls.stateDir}/gateway.pem ${svcCert}
|
||||
install -m 0640 -g nginx ${config.services.hyperhive.deploy.hive-controller.tls.stateDir}/gateway-key.pem ${svcKey}
|
||||
fi
|
||||
'';
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@
|
|||
matrixCfg, # services.hyperhive.swarm.matrix — what the homeserver IS
|
||||
matrixDeployCfg, # services.hyperhive.deploy.matrix — what THIS host runs
|
||||
hyperhiveDomain,
|
||||
runsHive, # deploy.hive-controller.enable — whether hive-c0re runs here
|
||||
dashboardDist,
|
||||
swaggerUiTheme, # nix/packages/swagger-ui-theme.nix: has index.html + hyperhive-theme.css
|
||||
errorPages, # ./error-pages.nix: { notFound, unreachable, unauthorized, ssoUnavailable }
|
||||
|
|
@ -246,13 +247,12 @@ in
|
|||
return 444;
|
||||
'';
|
||||
};
|
||||
|
||||
# The hive's own surface, now reachable by NAME. This used to be
|
||||
# served by the `_` vhost above: no vhost was named for the hive
|
||||
# domain, so every dashboard and agent-UI request matched the
|
||||
# default server instead (confirmed against 24h of nginx's access
|
||||
# log — `server: _` on requests whose Host *was* the hive domain).
|
||||
# Naming it is what lets the catch-all start rejecting.
|
||||
}
|
||||
// lib.optionalAttrs runsHive {
|
||||
# The hive's own surface, reachable by NAME. Named rather than left
|
||||
# to `_`, so the catch-all can reject everything else. Only where a
|
||||
# hive runs: it proxies to hive-c0re, which runs only with the hive
|
||||
# controller.
|
||||
${hyperhiveDomain} = (vhostTlsFor hyperhiveDomain) // {
|
||||
listen = vhostListen;
|
||||
locations =
|
||||
|
|
|
|||
|
|
@ -40,17 +40,15 @@ let
|
|||
# host's nginx also carries the stream passthrough every reader dials to
|
||||
# reach the store, and waiting on a login to that store would take that
|
||||
# path down whenever it is sealed. There the gateway starts on the hive
|
||||
# leaf fallback, and the issuing script re-imports the services leaf into
|
||||
# the running nginx once it lands.
|
||||
# leaf, or on a host without a hive on a throwaway self-signed pair
|
||||
# (../hive-gateway/default.nix), and the issuing script re-imports the
|
||||
# services leaf into the running nginx once it lands.
|
||||
#
|
||||
# Nor on a host fronting no swarm name: it asks the store for the root
|
||||
# alone, which nginx never reads, and a sealed store would otherwise hold
|
||||
# its gateway down.
|
||||
#
|
||||
# On a host with no hive there is no hive leaf to start on: the services
|
||||
# leaf is the only certificate its gateway has, so it always waits.
|
||||
gatewayWaitsForServicesLeaf =
|
||||
localServiceDomains != [ ] && (!runsHive || localServiceDomains != [ baoCfg.ui.domain ]);
|
||||
localServiceDomains != [ ] && localServiceDomains != [ baoCfg.ui.domain ];
|
||||
|
||||
# The hive leaf, `hive-tls-resign` and the hive identity it needs exist only
|
||||
# where a hive runs. A gateway host without one serves the swarm-services
|
||||
|
|
|
|||
|
|
@ -38,6 +38,13 @@ let
|
|||
deploy.bao.enable = false;
|
||||
};
|
||||
|
||||
# A dedicated secret-store host: no hive, and the store's browser UI the
|
||||
# only swarm name its gateway fronts.
|
||||
storeOnlyNoHive = hive {
|
||||
deploy.hive-controller.enable = false;
|
||||
deploy.bao.enable = true;
|
||||
};
|
||||
|
||||
# The same switch with every service it owns placed elsewhere, so the only
|
||||
# thing that can enable the gateway/resolver/bridge here is that toggle's
|
||||
# own module — each service module asserts them from its own `enable`.
|
||||
|
|
@ -196,34 +203,88 @@ let
|
|||
&& swarmServicesSwitchAlone.services.hyperhive.network.enable;
|
||||
}
|
||||
{
|
||||
# No hive, so no hive leaf and no hive identity: the gateway's default
|
||||
# certificate is the swarm-services leaf, and its start waits for that
|
||||
# leaf. The control is a hive host with the same services, which keeps
|
||||
# the hive leaf and the unit that issues it.
|
||||
# No hive, so no hive leaf, no hive identity and no hive-domain vhost:
|
||||
# the default certificate is the swarm-services leaf, or a throwaway
|
||||
# pair until it lands. A host fronting many swarm names waits for that
|
||||
# leaf; a store-only host does not. The control is a hive host with
|
||||
# the same services.
|
||||
name = "a gateway host without a hive serves the swarm-services leaf as its default cert and has no hive-leaf unit";
|
||||
ok =
|
||||
let
|
||||
# Every hive-leaf reference a unit could carry: the leaf files,
|
||||
# the identity's file and subject. The store's own per-hive role
|
||||
# unit is left out: it writes roles for the hives in the swarm
|
||||
# directory, which live elsewhere.
|
||||
hiveLeafRefs = [
|
||||
"gateway.pem"
|
||||
"gateway-key.pem"
|
||||
"hive-leaf"
|
||||
"swarm-bao-hive-leaf-"
|
||||
];
|
||||
noHiveLeafAnywhere =
|
||||
m:
|
||||
let
|
||||
units = removeAttrs m.systemd.services [ "swarm-bao-hive-leaf-policy" ];
|
||||
texts = lib.concatMap (u: [
|
||||
(u.script or "")
|
||||
(builtins.toJSON (u.environment or { }))
|
||||
]) (lib.attrValues units);
|
||||
in
|
||||
!(lib.any (t: lib.any (r: lib.hasInfix r t) hiveLeafRefs) texts)
|
||||
&& m.services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile == null
|
||||
&& !(m.services.nginx.virtualHosts ? "h1.t.local");
|
||||
noHive =
|
||||
m:
|
||||
let
|
||||
s = m.systemd.services;
|
||||
imp = s.hive-gateway-self-signed-cert.script;
|
||||
in
|
||||
!(s ? hive-tls-resign)
|
||||
&& !(m.systemd.timers ? hive-tls-resign)
|
||||
&& lib.hasInfix "install -m 0644 /var/lib/hive-tls/swarm-services.pem /var/lib/hive-gateway/tls/cert.pem" imp
|
||||
&& !(lib.hasInfix "/var/lib/hive-tls/gateway.pem /var/lib/hive-gateway/tls/cert.pem" imp)
|
||||
&& lib.elem "hive-gateway-self-signed-cert.service" s.swarm-services-cert.requiredBy
|
||||
&& lib.elem "hive-gateway-self-signed-cert.service" s.swarm-services-cert.before
|
||||
# Fronting swarm names, so the root-only branch that presents the
|
||||
# hive identity is not the one that runs.
|
||||
&& lib.hasInfix "want_svc=1" s.swarm-services-cert.script;
|
||||
!(s ? hive-tls-resign) && !(m.systemd.timers ? hive-tls-resign) && noHiveLeafAnywhere m;
|
||||
waitsForLeaf =
|
||||
m:
|
||||
let
|
||||
s = m.systemd.services;
|
||||
in
|
||||
lib.elem "hive-gateway-self-signed-cert.service" s.swarm-services-cert.requiredBy
|
||||
&& lib.elem "hive-gateway-self-signed-cert.service" s.swarm-services-cert.before;
|
||||
# The cert it installs on the default and swarm-services paths:
|
||||
# the services leaf when present, a throwaway pair otherwise.
|
||||
importsServicesLeafOrFallback =
|
||||
m:
|
||||
let
|
||||
imp = m.systemd.services.hive-gateway-self-signed-cert;
|
||||
in
|
||||
lib.hasInfix "cert=/var/lib/hive-tls/swarm-services.pem" imp.script
|
||||
&& lib.hasInfix "openssl req -x509" imp.script
|
||||
&& lib.hasInfix "cert=/var/lib/hive-gateway/tls/fallback.pem" imp.script
|
||||
&& lib.hasInfix ''install -m 0644 "$cert" /var/lib/hive-gateway/tls/cert.pem'' imp.script
|
||||
&& lib.hasInfix ''install -m 0644 "$cert" /var/lib/hive-gateway/tls/swarm-services.pem'' imp.script
|
||||
&& lib.any (p: lib.hasPrefix "openssl" (p.pname or (p.name or ""))) imp.path;
|
||||
store = storeOnlyNoHive;
|
||||
hs = swarmServicesHere.systemd.services;
|
||||
in
|
||||
noHive swarmServicesOnly
|
||||
&& noHive swarmServicesOnlyNoStore
|
||||
&& noHive store
|
||||
&& importsServicesLeafOrFallback swarmServicesOnly
|
||||
&& importsServicesLeafOrFallback swarmServicesOnlyNoStore
|
||||
&& importsServicesLeafOrFallback store
|
||||
# Fronting many swarm names: the gateway waits for their leaf.
|
||||
&& waitsForLeaf swarmServicesOnly
|
||||
&& waitsForLeaf swarmServicesOnlyNoStore
|
||||
# The store's UI alone: its nginx carries the passthrough to the
|
||||
# store, so it starts on the fallback with the store sealed.
|
||||
&&
|
||||
store.services.hyperhive.swarm.localServiceDomains
|
||||
== [ store.services.hyperhive.swarm.bao.ui.domain ]
|
||||
&& !(waitsForLeaf store)
|
||||
&& lib.hasInfix "want_svc=1" store.systemd.services.swarm-services-cert.script
|
||||
# Control: a hive host with the same services keeps the hive leaf,
|
||||
# its unit and identity, and the hive-domain vhost.
|
||||
&& hs ? hive-tls-resign
|
||||
&& lib.hasInfix "install -m 0644 /var/lib/hive-tls/gateway.pem /var/lib/hive-gateway/tls/cert.pem" hs.hive-gateway-self-signed-cert.script;
|
||||
&& lib.hasInfix "install -m 0644 /var/lib/hive-tls/gateway.pem /var/lib/hive-gateway/tls/cert.pem" hs.hive-gateway-self-signed-cert.script
|
||||
&& swarmServicesHere.services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile != null
|
||||
&& swarmServicesHere.services.nginx.virtualHosts ? "h1.t.local"
|
||||
&& !(noHiveLeafAnywhere swarmServicesHere);
|
||||
}
|
||||
{
|
||||
# Each service is gated on its own deployment toggle alone, so a host
|
||||
|
|
|
|||
Loading…
Reference in a new issue