diff --git a/docs/networking/gateway.md b/docs/networking/gateway.md index 36e1c2d3..21b39f1b 100644 --- a/docs/networking/gateway.md +++ b/docs/networking/gateway.md @@ -74,9 +74,9 @@ nginx obtains and renews certs via the HTTP-01 challenge on `port` (default 80); On by default, listening on `httpsPort` (default 443) beside the plain-http `port` (default 80). -The issuer is the **secret store's `pki` mount**, not a bare self-signed leaf. `hive-tls-resign.service` (from `hive-tls.nix`) asks the store for a gateway **leaf** through this hive's own role, presenting the hive's own store identity (`services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile` / `baoHiveIdentityKeyFile`), and writes it under `…tls.stateDir` (default `/var/lib/hive-tls`). `hive-gateway-self-signed-cert` then imports the leaf into nginx's state dir (`/var/lib/hive-gateway/tls/{cert,key}.pem`). [`swarm/ca.md`](../swarm/ca.md) has the role, the identity and what a hive without one does. +The issuer is the **secret store's `pki` mount**, not a bare self-signed leaf. `hive-tls-resign.service` (from `hive-tls.nix`) asks the store for a gateway **leaf** through this hive's own role, presenting the hive's own store identity (`services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile` / `baoHiveIdentityKeyFile`), and writes it under `…tls.stateDir` (default `/var/lib/hive-tls`). `hive-gateway-self-signed-cert` then imports the leaf into nginx's state dir (`/var/lib/hive-gateway/tls/{cert,key}.pem`). On a host that runs no hive the import unit installs the swarm-services leaf there instead, or a throwaway self-signed pair until that leaf lands. [`swarm/ca.md`](../swarm/ca.md) has the role, the identity and what a hive without one does. -⚠️ **Keep the import unit.** It does two jobs nginx needs: it re-modes the key to `0640 root:nginx` (nginx's pre-start `nginx -t` runs as the nginx user and fails on the source's `0600 root:root` key), and it makes sure **every cert path the config names exists** — when the swarm-services leaf is missing it installs the hive leaf in its place. nginx refuses a config naming a missing cert file, so without that fallback one missing leaf takes down every vhost, not just one. +⚠️ **Keep the import unit.** It does two jobs nginx needs: it re-modes the key to `0640 root:nginx` (nginx's pre-start `nginx -t` runs as the nginx user and fails on the source's `0600 root:root` key), and it makes sure **every cert path the config names exists** — when the swarm-services leaf is missing it installs the hive leaf in its place, or on a host that runs no hive a throwaway self-signed pair. nginx refuses a config naming a missing cert file, so without that fallback one missing leaf takes down every vhost, not just one. **Why an issuer, not a bare leaf**: a bare self-signed leaf is its own trust anchor, so every regeneration is a new anchor every consumer must re-trust, and an agent's build sets its trust store once, before any leaf exists. Agents and federation peers trust the store's root once; leaf rotation never breaks them. diff --git a/docs/swarm/ca.md b/docs/swarm/ca.md index 2e9683aa..f9f9c2f1 100644 --- a/docs/swarm/ca.md +++ b/docs/swarm/ca.md @@ -64,10 +64,13 @@ nginx serves the leaf alone, without its issuer: the issuer is the self-signed root a consumer already holds in its trust bundle. **A gateway host that runs no hive** (`deploy.hive-controller.enable` off, -such as a dedicated swarm-services host) has no hive leaf, no -`hive-tls-resign.service` and needs no hive identity. Its gateway serves the -swarm-services leaf as its default certificate too, and waits for -`swarm-services-cert.service` at start. +such as a dedicated swarm-services or secret-store host) has no hive leaf, no +`hive-tls-resign.service`, no hive identity and no hive-domain vhost. Its +gateway serves the swarm-services leaf as its default certificate too. Where it +fronts other swarm names it waits for `swarm-services-cert.service` at start. +Where the store's UI is its only swarm name it starts on a throwaway +self-signed certificate and switches to the services leaf once the store +issues it. ## The trust bundle diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index 3c33c1f8..62a01a02 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -447,8 +447,9 @@ credential. Anything on the store's host that can dial loopback, and any `admins` session through the vhost, needs only a token to use the API. Unseal from the host's `bao` CLI; the UI's unseal form gets a 403. On a self-signed gateway where the UI is the only swarm name its host fronts, that host's nginx -doesn't wait for the store: it serves the hive certificate on the UI's name (a -browser warning) until the unsealed store issues the services leaf, so the +doesn't wait for the store: it serves the hive leaf, or on a host that runs no +hive a throwaway self-signed certificate, on the UI's name (a browser warning) +until the unsealed store issues the services leaf, so the stream passthrough readers use on that host comes up with the store sealed. ### OIDC login diff --git a/nix/host-modules/glue-bao-tls.nix b/nix/host-modules/glue-bao-tls.nix index 51241ab6..8577ee93 100644 --- a/nix/host-modules/glue-bao-tls.nix +++ b/nix/host-modules/glue-bao-tls.nix @@ -262,13 +262,16 @@ in [ -s ${pkiDir}/nats-auth.pem ] || ${signLeaf} ${pkiDir} nats-auth \ ${lib.escapeShellArg deployCfg.bao.natsAuthCommonName} "" clientAuth - # This hive's gateway-leaf identity, which opens - # `pki/issue/-` and nothing else. Its - # subject carries this hive's name, like the matrix-token and - # queue-agent leaves above, because ./swarm-bao.nix writes one role per - # hive. Hives on other hosts are issued theirs out of band. - [ -s ${pkiDir}/hive-leaf.pem ] || ${signLeaf} ${pkiDir} hive-leaf \ - ${lib.escapeShellArg "${deployCfg.bao.hiveLeafCommonNamePrefix}-${clientCn}"} "" clientAuth + ${lib.optionalString deployCfg.hive-controller.enable '' + # This hive's gateway-leaf identity, which opens + # `pki/issue/-` and nothing else. Its + # subject carries this hive's name, like the matrix-token and + # queue-agent leaves above, because ./swarm-bao.nix writes one role + # per hive. Only where a hive runs; hives on other hosts are issued + # theirs out of band. + [ -s ${pkiDir}/hive-leaf.pem ] || ${signLeaf} ${pkiDir} hive-leaf \ + ${lib.escapeShellArg "${deployCfg.bao.hiveLeafCommonNamePrefix}-${clientCn}"} "" clientAuth + ''} ''; }; }; diff --git a/nix/host-modules/glue-hive-tls-bao-identity.nix b/nix/host-modules/glue-hive-tls-bao-identity.nix index a27dfb6a..f3a13880 100644 --- a/nix/host-modules/glue-hive-tls-bao-identity.nix +++ b/nix/host-modules/glue-hive-tls-bao-identity.nix @@ -8,7 +8,8 @@ # ⚠️ The minting is NOT here. ./glue-bao-tls.nix holds the CA and signs the # leaf, for the reason ./glue-services-issuer-bao-identity.nix gives. # -# Gated on the minted PKI existing, as that file is. +# Gated on the minted PKI existing, as that file is, and on a hive running +# here: a host without one has no hive leaf to ask for. # # Everything is `mkDefault`. An operator naming their own paths wins. { @@ -25,7 +26,7 @@ let pkiDir = if haveMintedPki then builtins.dirOf baoDeploy.clientCertFile else null; in { - config = lib.mkIf haveMintedPki { + config = lib.mkIf (haveMintedPki && config.services.hyperhive.deploy.hive-controller.enable) { services.hyperhive.deploy.hive-controller.tls = { baoHiveIdentityCertFile = lib.mkDefault "${pkiDir}/hive-leaf.pem"; baoHiveIdentityKeyFile = lib.mkDefault "${pkiDir}/hive-leaf-key.pem"; diff --git a/nix/host-modules/hive-gateway/default.nix b/nix/host-modules/hive-gateway/default.nix index 7ad01445..8a8d1e27 100644 --- a/nix/host-modules/hive-gateway/default.nix +++ b/nix/host-modules/hive-gateway/default.nix @@ -113,6 +113,7 @@ let matrixCfg matrixDeployCfg hyperhiveDomain + runsHive dashboardDist swaggerUiTheme ; @@ -276,7 +277,8 @@ in # Pin the journal identity (else it's the `script` store-path wrapper). SyslogIdentifier = "hive-gateway-self-signed-cert"; }; - path = [ pkgs.coreutils ]; + # openssl for the throwaway pair a host without a hive starts on. + path = [ pkgs.coreutils ] ++ lib.optional (!runsHive) pkgs.openssl; script = '' set -eu mkdir -p ${tlsDir} @@ -301,45 +303,62 @@ in if runsHive then '' install -m 0644 ${tlsStateDir}/gateway.pem ${tlsCert} - install -m 0640 -g nginx ${tlsStateDir}/gateway-key.pem ${tlsKey}'' + install -m 0640 -g nginx ${tlsStateDir}/gateway-key.pem ${tlsKey} + + # The swarm-services leaf, when this host issues one. It is + # a separate pair rather than more SANs on the one above + # because the hive leaf's role allows only its own hive's + # domain, and the service names are siblings of it. + # + # Absent is a normal state, not a failure: a host fronting + # no swarm name has none, and issuance can also fail on a + # host that wants one. + # + # ⚠️ When it is absent the HIVE leaf goes to this path + # anyway, and that fallback is load-bearing rather than + # tidy. nginx refuses to load a config naming a cert file + # that does not exist — `cannot load certificate … no such + # file` fails the pre-start test, so the vhost does not + # degrade, the ENTIRE gateway dies and takes the forge, the + # dashboard and matrix with it. Serving the hive leaf on a + # swarm-service name is a name mismatch: browsers warn, + # strict clients refuse, everything else keeps working, and + # the operator gets a bad cert instead of no hive. + # + # Measured, not theorised: this exact path took pr1ma's + # gateway down when the services sub-CA failed to issue. + if [ -s ${tlsStateDir}/swarm-services.pem ]; then + install -m 0644 ${tlsStateDir}/swarm-services.pem ${svcCert} + install -m 0640 -g nginx ${tlsStateDir}/swarm-services-key.pem ${svcKey} + else + echo "no swarm-services leaf — serving the hive leaf on those names (mismatch, not an outage)" >&2 + install -m 0644 ${tlsStateDir}/gateway.pem ${svcCert} + install -m 0640 -g nginx ${tlsStateDir}/gateway-key.pem ${svcKey} + fi'' else '' - # No hive here, so no hive leaf: the catch-all and any other - # vhost on the hive cert path serve the swarm-services leaf. - install -m 0644 ${tlsStateDir}/swarm-services.pem ${tlsCert} - install -m 0640 -g nginx ${tlsStateDir}/swarm-services-key.pem ${tlsKey}'' + # No hive here, so no hive leaf: every vhost, the catch-all + # included, serves the swarm-services leaf. Until it lands a + # throwaway self-signed pair stands in, because nginx refuses + # a config naming a missing cert file. A store-only host does + # not wait for the leaf, so its stream passthrough comes up with + # the store sealed. `swarm-services-cert` re-runs this unit + # once the leaf lands. + if [ -s ${tlsStateDir}/swarm-services.pem ] && [ -s ${tlsStateDir}/swarm-services-key.pem ]; then + cert=${tlsStateDir}/swarm-services.pem + key=${tlsStateDir}/swarm-services-key.pem + else + echo "no swarm-services leaf yet — serving a throwaway self-signed certificate until it lands" >&2 + cert=${tlsDir}/fallback.pem + key=${tlsDir}/fallback-key.pem + (umask 077; openssl req -x509 -newkey rsa:2048 -nodes -sha256 -days 30 \ + -subj "/CN=hyperhive-gateway-fallback" -keyout "$key" -out "$cert") + fi + install -m 0644 "$cert" ${tlsCert} + install -m 0640 -g nginx "$key" ${tlsKey} + install -m 0644 "$cert" ${svcCert} + install -m 0640 -g nginx "$key" ${svcKey}'' } - - # The swarm-services leaf, when this host issues one. It is - # a separate pair rather than more SANs on the one above - # because the hive leaf's role allows only its own hive's - # domain, and the service names are siblings of it. - # - # Absent is a normal state, not a failure: a host fronting - # no swarm name has none, and issuance can also fail on a - # host that wants one. - # - # ⚠️ When it is absent the HIVE leaf goes to this path - # anyway, and that fallback is load-bearing rather than - # tidy. nginx refuses to load a config naming a cert file - # that does not exist — `cannot load certificate … no such - # file` fails the pre-start test, so the vhost does not - # degrade, the ENTIRE gateway dies and takes the forge, the - # dashboard and matrix with it. Serving the hive leaf on a - # swarm-service name is a name mismatch: browsers warn, - # strict clients refuse, everything else keeps working, and - # the operator gets a bad cert instead of no hive. - # - # Measured, not theorised: this exact path took pr1ma's - # gateway down when the services sub-CA failed to issue. - if [ -s ${config.services.hyperhive.deploy.hive-controller.tls.stateDir}/swarm-services.pem ]; then - install -m 0644 ${config.services.hyperhive.deploy.hive-controller.tls.stateDir}/swarm-services.pem ${svcCert} - install -m 0640 -g nginx ${config.services.hyperhive.deploy.hive-controller.tls.stateDir}/swarm-services-key.pem ${svcKey} - else - echo "no swarm-services leaf — serving the hive leaf on those names (mismatch, not an outage)" >&2 - install -m 0644 ${config.services.hyperhive.deploy.hive-controller.tls.stateDir}/gateway.pem ${svcCert} - install -m 0640 -g nginx ${config.services.hyperhive.deploy.hive-controller.tls.stateDir}/gateway-key.pem ${svcKey} - fi ''; }; diff --git a/nix/host-modules/hive-gateway/vhosts.nix b/nix/host-modules/hive-gateway/vhosts.nix index fa46a723..a1413ace 100644 --- a/nix/host-modules/hive-gateway/vhosts.nix +++ b/nix/host-modules/hive-gateway/vhosts.nix @@ -18,6 +18,7 @@ matrixCfg, # services.hyperhive.swarm.matrix — what the homeserver IS matrixDeployCfg, # services.hyperhive.deploy.matrix — what THIS host runs hyperhiveDomain, + runsHive, # deploy.hive-controller.enable — whether hive-c0re runs here dashboardDist, swaggerUiTheme, # nix/packages/swagger-ui-theme.nix: has index.html + hyperhive-theme.css errorPages, # ./error-pages.nix: { notFound, unreachable, unauthorized, ssoUnavailable } @@ -246,13 +247,12 @@ in return 444; ''; }; - - # The hive's own surface, now reachable by NAME. This used to be - # served by the `_` vhost above: no vhost was named for the hive - # domain, so every dashboard and agent-UI request matched the - # default server instead (confirmed against 24h of nginx's access - # log — `server: _` on requests whose Host *was* the hive domain). - # Naming it is what lets the catch-all start rejecting. + } + // lib.optionalAttrs runsHive { + # The hive's own surface, reachable by NAME. Named rather than left + # to `_`, so the catch-all can reject everything else. Only where a + # hive runs: it proxies to hive-c0re, which runs only with the hive + # controller. ${hyperhiveDomain} = (vhostTlsFor hyperhiveDomain) // { listen = vhostListen; locations = diff --git a/nix/host-modules/hive-tls.nix b/nix/host-modules/hive-tls.nix index 79a709af..0140580d 100644 --- a/nix/host-modules/hive-tls.nix +++ b/nix/host-modules/hive-tls.nix @@ -40,17 +40,15 @@ let # host's nginx also carries the stream passthrough every reader dials to # reach the store, and waiting on a login to that store would take that # path down whenever it is sealed. There the gateway starts on the hive - # leaf fallback, and the issuing script re-imports the services leaf into - # the running nginx once it lands. + # leaf, or on a host without a hive on a throwaway self-signed pair + # (../hive-gateway/default.nix), and the issuing script re-imports the + # services leaf into the running nginx once it lands. # # Nor on a host fronting no swarm name: it asks the store for the root # alone, which nginx never reads, and a sealed store would otherwise hold # its gateway down. - # - # On a host with no hive there is no hive leaf to start on: the services - # leaf is the only certificate its gateway has, so it always waits. gatewayWaitsForServicesLeaf = - localServiceDomains != [ ] && (!runsHive || localServiceDomains != [ baoCfg.ui.domain ]); + localServiceDomains != [ ] && localServiceDomains != [ baoCfg.ui.domain ]; # The hive leaf, `hive-tls-resign` and the hive identity it needs exist only # where a hive runs. A gateway host without one serves the swarm-services diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix index 242295b9..2dc11780 100644 --- a/nix/module-eval/swarm-services-switch.nix +++ b/nix/module-eval/swarm-services-switch.nix @@ -38,6 +38,13 @@ let deploy.bao.enable = false; }; + # A dedicated secret-store host: no hive, and the store's browser UI the + # only swarm name its gateway fronts. + storeOnlyNoHive = hive { + deploy.hive-controller.enable = false; + deploy.bao.enable = true; + }; + # The same switch with every service it owns placed elsewhere, so the only # thing that can enable the gateway/resolver/bridge here is that toggle's # own module — each service module asserts them from its own `enable`. @@ -196,34 +203,88 @@ let && swarmServicesSwitchAlone.services.hyperhive.network.enable; } { - # No hive, so no hive leaf and no hive identity: the gateway's default - # certificate is the swarm-services leaf, and its start waits for that - # leaf. The control is a hive host with the same services, which keeps - # the hive leaf and the unit that issues it. + # No hive, so no hive leaf, no hive identity and no hive-domain vhost: + # the default certificate is the swarm-services leaf, or a throwaway + # pair until it lands. A host fronting many swarm names waits for that + # leaf; a store-only host does not. The control is a hive host with + # the same services. name = "a gateway host without a hive serves the swarm-services leaf as its default cert and has no hive-leaf unit"; ok = let + # Every hive-leaf reference a unit could carry: the leaf files, + # the identity's file and subject. The store's own per-hive role + # unit is left out: it writes roles for the hives in the swarm + # directory, which live elsewhere. + hiveLeafRefs = [ + "gateway.pem" + "gateway-key.pem" + "hive-leaf" + "swarm-bao-hive-leaf-" + ]; + noHiveLeafAnywhere = + m: + let + units = removeAttrs m.systemd.services [ "swarm-bao-hive-leaf-policy" ]; + texts = lib.concatMap (u: [ + (u.script or "") + (builtins.toJSON (u.environment or { })) + ]) (lib.attrValues units); + in + !(lib.any (t: lib.any (r: lib.hasInfix r t) hiveLeafRefs) texts) + && m.services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile == null + && !(m.services.nginx.virtualHosts ? "h1.t.local"); noHive = m: let s = m.systemd.services; - imp = s.hive-gateway-self-signed-cert.script; in - !(s ? hive-tls-resign) - && !(m.systemd.timers ? hive-tls-resign) - && lib.hasInfix "install -m 0644 /var/lib/hive-tls/swarm-services.pem /var/lib/hive-gateway/tls/cert.pem" imp - && !(lib.hasInfix "/var/lib/hive-tls/gateway.pem /var/lib/hive-gateway/tls/cert.pem" imp) - && lib.elem "hive-gateway-self-signed-cert.service" s.swarm-services-cert.requiredBy - && lib.elem "hive-gateway-self-signed-cert.service" s.swarm-services-cert.before - # Fronting swarm names, so the root-only branch that presents the - # hive identity is not the one that runs. - && lib.hasInfix "want_svc=1" s.swarm-services-cert.script; + !(s ? hive-tls-resign) && !(m.systemd.timers ? hive-tls-resign) && noHiveLeafAnywhere m; + waitsForLeaf = + m: + let + s = m.systemd.services; + in + lib.elem "hive-gateway-self-signed-cert.service" s.swarm-services-cert.requiredBy + && lib.elem "hive-gateway-self-signed-cert.service" s.swarm-services-cert.before; + # The cert it installs on the default and swarm-services paths: + # the services leaf when present, a throwaway pair otherwise. + importsServicesLeafOrFallback = + m: + let + imp = m.systemd.services.hive-gateway-self-signed-cert; + in + lib.hasInfix "cert=/var/lib/hive-tls/swarm-services.pem" imp.script + && lib.hasInfix "openssl req -x509" imp.script + && lib.hasInfix "cert=/var/lib/hive-gateway/tls/fallback.pem" imp.script + && lib.hasInfix ''install -m 0644 "$cert" /var/lib/hive-gateway/tls/cert.pem'' imp.script + && lib.hasInfix ''install -m 0644 "$cert" /var/lib/hive-gateway/tls/swarm-services.pem'' imp.script + && lib.any (p: lib.hasPrefix "openssl" (p.pname or (p.name or ""))) imp.path; + store = storeOnlyNoHive; hs = swarmServicesHere.systemd.services; in noHive swarmServicesOnly && noHive swarmServicesOnlyNoStore + && noHive store + && importsServicesLeafOrFallback swarmServicesOnly + && importsServicesLeafOrFallback swarmServicesOnlyNoStore + && importsServicesLeafOrFallback store + # Fronting many swarm names: the gateway waits for their leaf. + && waitsForLeaf swarmServicesOnly + && waitsForLeaf swarmServicesOnlyNoStore + # The store's UI alone: its nginx carries the passthrough to the + # store, so it starts on the fallback with the store sealed. + && + store.services.hyperhive.swarm.localServiceDomains + == [ store.services.hyperhive.swarm.bao.ui.domain ] + && !(waitsForLeaf store) + && lib.hasInfix "want_svc=1" store.systemd.services.swarm-services-cert.script + # Control: a hive host with the same services keeps the hive leaf, + # its unit and identity, and the hive-domain vhost. && hs ? hive-tls-resign - && lib.hasInfix "install -m 0644 /var/lib/hive-tls/gateway.pem /var/lib/hive-gateway/tls/cert.pem" hs.hive-gateway-self-signed-cert.script; + && lib.hasInfix "install -m 0644 /var/lib/hive-tls/gateway.pem /var/lib/hive-gateway/tls/cert.pem" hs.hive-gateway-self-signed-cert.script + && swarmServicesHere.services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile != null + && swarmServicesHere.services.nginx.virtualHosts ? "h1.t.local" + && !(noHiveLeafAnywhere swarmServicesHere); } { # Each service is gated on its own deployment toggle alone, so a host