From 22b9c570ba76aa2c23ebd112e992ae52826fa1ca Mon Sep 17 00:00:00 2001 From: atlas Date: Sat, 17 Oct 2026 18:48:52 +0200 Subject: [PATCH] hive-tls owns the per-host list of locally fronted service domains --- docs/swarm/ca.md | 2 +- nix/host-modules/hive-tls.nix | 1155 +++++++++++---------- nix/host-modules/swarm-bao.nix | 2 +- nix/host-modules/swarm.nix | 27 +- nix/module-eval/bao-basics.nix | 2 +- nix/module-eval/core-toggle.nix | 2 +- nix/module-eval/hive-tls.nix | 4 +- nix/module-eval/nats-tls.nix | 2 +- nix/module-eval/swarm-services-switch.nix | 2 +- 9 files changed, 603 insertions(+), 595 deletions(-) diff --git a/docs/swarm/ca.md b/docs/swarm/ca.md index 0739f7da..3019fd22 100644 --- a/docs/swarm/ca.md +++ b/docs/swarm/ca.md @@ -146,7 +146,7 @@ same list on every host, populates the role's `allowed_domains`, so asking for a name nobody configured is a refusal from the store naming that name — not a certificate quietly issued for it. A host's own SANs are narrower still: it asks only for the names it -fronts a vhost for (`swarm.localServiceDomains`), so the key on one +fronts a vhost for (`deploy.hive-controller.tls.localServiceDomains`), so the key on one gateway can't serve a swarm service that runs behind another. **The root's public certificate is a file, on every self-signed gateway host:** diff --git a/nix/host-modules/hive-tls.nix b/nix/host-modules/hive-tls.nix index 04fda83b..16679cc7 100644 --- a/nix/host-modules/hive-tls.nix +++ b/nix/host-modules/hive-tls.nix @@ -20,11 +20,11 @@ let baoServicesPkiMount = baoDeploy.servicesPkiMountPath; baoServicesPkiRole = baoDeploy.servicesPkiRoleName; - # Derived once in ./swarm.nix: the swarm service names THIS host fronts - # a vhost for. Every one of them is inside the swarm-wide set that - # ./swarm-bao.nix narrows `pki/roles/swarm-services` to, so the leaf - # stays issuable while carrying no name this gateway does not serve. - localServiceDomains = hyperhiveCfg.swarm.localServiceDomains; + # The swarm service names THIS host fronts a vhost for. Every one of them + # is inside the swarm-wide set that ./swarm-bao.nix narrows + # `pki/roles/swarm-services` to, so the leaf stays issuable while + # carrying no name this gateway does not serve. + localServiceDomains = cfg.localServiceDomains; # The one name the request's `common_name` carries; the rest ride as # SANs. Not `builtins.head`: a host fronting no swarm service renders @@ -346,588 +346,621 @@ in Both or neither — a certificate with no key authenticates nothing. ''; }; + + localServiceDomains = lib.mkOption { + type = lib.types.listOf lib.types.str; + readOnly = true; + internal = true; + description = '' + Read-only: the names in + {option}`services.hyperhive.swarm.serviceDomains` **this host + actually fronts**, which is what its services leaf may carry. The + swarm-wide set stays what the sub-CA is name-constrained to and what + the store's issuing role permits; narrowing the leaf is what stops a + private key on one gateway presenting a valid certificate for + services that host does not serve and never has. + + Read off the rendered `services.nginx.virtualHosts` rather than off + the per-service deploy flags: those flags are what the vhosts are + already guarded on, so a second reading of them is a copy that + drifts from the thing it claims to describe. Filtering + `swarm.serviceDomains` rather than assembling a list beside it is + that option's own constraint, and it applies here for the same + reason. + ''; + }; }; - config = lib.mkIf active { - # The state dir and the trust bundle. Never waits on the store: the - # store's container is ordered after this unit, which writes the - # bundle it bind-mounts, so the leaf is `hive-tls-resign`'s. - systemd.services.hive-tls-ca = { - description = "Assemble the hive TLS trust bundle (self-signed mode)"; - wantedBy = [ "multi-user.target" ]; - # The consumer is `hive-gateway-self-signed-cert`, which copies the - # leaf out of the state dir this unit creates, and which nginx in - # turn `Requires=`. Order against the unit that reads the dir. - before = [ "hive-gateway-self-signed-cert.service" ]; - requiredBy = [ "hive-gateway-self-signed-cert.service" ]; - # `swarm-services-cert` is NOT ordered against this unit in either - # direction, and the absence is load-bearing — see the cycle spelled - # out at that unit. It runs after the store's container, the container - # runs after this unit, so an edge back to here is a loop and systemd - # resolves a loop by dropping a job. - # - # What that costs: the trust bundle written at the end of this script - # is the services root, so on a first boot it is written before that - # file exists and is empty until a restart. The `if [ -s - # "$svcroot" ]` guard is what makes that state legal rather than fatal, - # and `swarm-services-cert` restarts this unit once the root it wrote - # is new — so the bundle converges instead of being a boot behind. - # - # ⚠️ A switch that changes this unit reloads it, running the same - # script, rather than stopping and starting it. A stop propagates to - # every unit that `Requires=` this one: the store's container, which - # comes back sealed under the `shamir` seal, every container that - # binds the bundle, and every `-ca-bundle` unit with its - # consumers. A reload rewrites the bundle in place and touches none of - # them, so a `-ca-bundle` unit keeps the copy it made at start - # until it restarts. - reloadIfChanged = true; - reload = config.systemd.services.hive-tls-ca.script; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - UMask = "0077"; - # Pin the journal identity (else it's the `script` store-path wrapper). - SyslogIdentifier = "hive-tls-ca"; - }; - script = '' - set -euo pipefail - d=${lib.escapeShellArg cfg.stateDir} - install -d -m 0755 "$d" + config = lib.mkMerge [ + { + # Outside `active`: the option has no default, so reading it on a host + # whose gateway is not self-signed would otherwise throw. + services.hyperhive.deploy.hive-controller.tls.localServiceDomains = lib.filter ( + d: config.services.nginx.virtualHosts ? ${d} + ) hyperhiveCfg.swarm.serviceDomains; + } - svcroot="$d/swarm-services-root.pem" - - # --- Trust bundle: what a consumer must TRUST, the store's `pki` - # root, which both the gateway leaf and the swarm-services leaf - # chain to (docs/swarm/ca.md). + (lib.mkIf active { + # The state dir and the trust bundle. Never waits on the store: the + # store's container is ordered after this unit, which writes the + # bundle it bind-mounts, so the leaf is `hive-tls-resign`'s. + systemd.services.hive-tls-ca = { + description = "Assemble the hive TLS trust bundle (self-signed mode)"; + wantedBy = [ "multi-user.target" ]; + # The consumer is `hive-gateway-self-signed-cert`, which copies the + # leaf out of the state dir this unit creates, and which nginx in + # turn `Requires=`. Order against the unit that reads the dir. + before = [ "hive-gateway-self-signed-cert.service" ]; + requiredBy = [ "hive-gateway-self-signed-cert.service" ]; + # `swarm-services-cert` is NOT ordered against this unit in either + # direction, and the absence is load-bearing — see the cycle spelled + # out at that unit. It runs after the store's container, the container + # runs after this unit, so an edge back to here is a loop and systemd + # resolves a loop by dropping a job. # - # Written IN PLACE, never renamed into position — containers - # bind-mount this file and a bind mount follows the inode, so a - # rename leaves every consumer holding the old one. - bundle="$d/trust-bundle.pem" - if [ -s "$svcroot" ]; then - cat "$svcroot" > "$bundle" - else - echo "the store's root $svcroot is missing or empty, so $bundle is empty and every *-ca-bundle unit fails until swarm-services-cert fetches it" >&2 - : > "$bundle" - fi - chmod 0644 "$bundle" - ''; - }; + # What that costs: the trust bundle written at the end of this script + # is the services root, so on a first boot it is written before that + # file exists and is empty until a restart. The `if [ -s + # "$svcroot" ]` guard is what makes that state legal rather than fatal, + # and `swarm-services-cert` restarts this unit once the root it wrote + # is new — so the bundle converges instead of being a boot behind. + # + # ⚠️ A switch that changes this unit reloads it, running the same + # script, rather than stopping and starting it. A stop propagates to + # every unit that `Requires=` this one: the store's container, which + # comes back sealed under the `shamir` seal, every container that + # binds the bundle, and every `-ca-bundle` unit with its + # consumers. A reload rewrites the bundle in place and touches none of + # them, so a `-ca-bundle` unit keeps the copy it made at start + # until it restarts. + reloadIfChanged = true; + reload = config.systemd.services.hive-tls-ca.script; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + UMask = "0077"; + # Pin the journal identity (else it's the `script` store-path wrapper). + SyslogIdentifier = "hive-tls-ca"; + }; + script = '' + set -euo pipefail + d=${lib.escapeShellArg cfg.stateDir} + install -d -m 0755 "$d" - # The gateway leaf: issued by the secret store's `pki` mount through - # this hive's own role, at boot, on every deploy and from the daily - # timer below, whenever it is missing, past half its lifetime, missing - # a configured name, not chained to the store's root, or not matching - # its key. - # - # ⚠️ Nothing orders the gateway against this unit, which is what lets a - # hive holding a valid leaf start its gateway with the store sealed or - # unreachable. Only a hive with no leaf yet has a gateway that waits - # for the store: its cert import fails until this unit lands a leaf - # and starts nginx. - # - # Failure leaves the current leaf alone: every file the script writes - # is a `.new` until the store has answered with both fields. - # - # Propagation mechanism: nginx serves a *copy* of the leaf, written by - # `hive-gateway-self-signed-cert` at the mode nginx can read. Issuing - # the source therefore changes nothing on its own — the copy has to be - # remade and nginx reloaded, which is what the calls at the end do. - # Neither of the running-nginx calls is `|| true`: a swallowed - # propagation failure means the leaf rotates on disk while nginx keeps - # serving the old copy until it expires, with this unit reporting - # success the whole time. - systemd.services.hive-tls-resign = lib.mkIf runsHive { - description = "Issue or renew the gateway TLS leaf from the secret store's PKI"; - wantedBy = [ "multi-user.target" ]; - # The store's container and its granting unit where they run here; - # ignored elsewhere, as for `swarm-services-cert`, which writes the - # store's root this unit checks the leaf against. - after = [ - "hive-tls-ca.service" - "container@${baoCfg.machine}.service" - "swarm-bao-pki.service" - "swarm-bao-hive-leaf-policy.service" - "swarm-services-cert.service" - ]; - wants = [ "container@${baoCfg.machine}.service" ]; - path = [ - baoDeploy.package - pkgs.jq - pkgs.openssl - pkgs.coreutils - pkgs.systemd - ]; - environment = hiveLeafEnvironment; - # ./lib/store-retry.nix: the login below fails for as long as the - # store is sealed or down. - inherit (storeRetry) startLimitBurst startLimitIntervalSec; - serviceConfig = storeRetry.serviceConfig // { - Type = "oneshot"; - UMask = "0077"; - SyslogIdentifier = "hive-tls-resign"; - # One attempt. A oneshot has no start timeout of its own, and an - # attempt that never ends never fails into a retry. - TimeoutStartSec = "10min"; - }; - script = '' - set -euo pipefail - d=${lib.escapeShellArg cfg.stateDir} - install -d -m 0755 "$d" - leaf="$d/gateway.pem" - leafk="$d/gateway-key.pem" + svcroot="$d/swarm-services-root.pem" - fail() { - echo "$1" >&2 - if [ -s "$leaf" ]; then - echo "the current gateway leaf at $leaf stays in place" >&2 + # --- Trust bundle: what a consumer must TRUST, the store's `pki` + # root, which both the gateway leaf and the swarm-services leaf + # chain to (docs/swarm/ca.md). + # + # Written IN PLACE, never renamed into position — containers + # bind-mount this file and a bind mount follows the inode, so a + # rename leaves every consumer holding the old one. + bundle="$d/trust-bundle.pem" + if [ -s "$svcroot" ]; then + cat "$svcroot" > "$bundle" + else + echo "the store's root $svcroot is missing or empty, so $bundle is empty and every *-ca-bundle unit fails until swarm-services-cert fetches it" >&2 + : > "$bundle" fi - exit 1 - } - - # Checked before the leaf is: a hive without its identity is red - # even while its current leaf is still valid. - ${ - if hiveLeafRole == null then - ''fail "services.hyperhive.hiveName is unset, so this hive has no gateway-leaf role to issue from."'' - else - requireHiveIdentity "ask the store for its gateway leaf" - } - - ${leafCoverage} - - # Half the lifetime the role grants, as for the services leaf, so - # the daily timer has half the lifetime to retry through a sealed - # store. - renewat=$(( ${toString baoDeploy.servicesPkiLeafTtlHours} * 3600 / 2 )) - reissue=0 - { [ -s "$leaf" ] && [ -s "$leafk" ]; } || reissue=1 - openssl x509 -in "$leaf" -noout -checkend "$renewat" >/dev/null 2>&1 || reissue=1 - leavesCoverNames || reissue=1 - # A leaf that does not chain to the store's current root, whoever - # signed it, fails this and is re-issued. - { [ -s "$svcroot" ] && openssl verify -CAfile "$svcroot" "$leaf" >/dev/null 2>&1; } || reissue=1 - # The leaf and the key are two renames, so an interrupted rotation can - # leave a certificate beside a key it does not belong to. - leafpub="$(openssl x509 -in "$leaf" -noout -pubkey 2>/dev/null || true)" - keypub="$(openssl pkey -in "$leafk" -pubout 2>/dev/null || true)" - { [ -n "$leafpub" ] && [ "$leafpub" = "$keypub" ]; } || reissue=1 - - if [ "$reissue" = 0 ]; then - echo "gateway leaf valid, matching its key, chained to the store's root and covering the configured names — leaving it alone" - exit 0 - fi - - err="$(mktemp)" - resp="$(mktemp "$d/gateway.json.XXXXXX")" - trap 'rm -f "$err" "$resp" "$leaf.new" "$leafk.new"' EXIT - - # Cert auth is a login, not a transport setting; `-token-only` - # skips the token helper, for the reason `swarm-services-cert` gives. - if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then - fail "could not log in to the swarm secret store with this hive's identity: $(cat "$err")" - fi - export BAO_TOKEN - - echo "requesting the gateway leaf from the store" - if ! bao write -format=json \ - ${lib.escapeShellArg "${baoServicesPkiMount}/issue/${toString hiveLeafRole}"} \ - common_name=${lib.escapeShellArg domain} \ - alt_names=${lib.escapeShellArg (lib.concatStringsSep "," (lib.tail hiveLeafNames))} \ - > "$resp" 2>"$err"; then - fail "the store refused to issue the gateway leaf: $(cat "$err")" - fi - - # One issuance, split, for the reasons `swarm-services-cert` gives: - # the key never enters an argv, and `// empty` keeps a missing field - # from becoming the four characters `null`. The leaf alone, without - # its issuer: the issuer is the self-signed root in the bundle. - umask 077 - jq -r '.data.private_key // empty' < "$resp" > "$leafk.new" - jq -r '.data.certificate // empty' < "$resp" > "$leaf.new" - for f in "$leafk.new" "$leaf.new"; do - [ -s "$f" ] || fail "the store's response was missing a field: $f is empty" - done - - mv -f "$leaf.new" "$leaf" - mv -f "$leafk.new" "$leafk" - chmod 0644 "$leaf" - chmod 0600 "$leafk" - - # A gateway already running gets the copy remade and a reload. One - # that is not running — a hive that had no leaf — gets started, - # `--no-block` so a start job still queued absorbs the request. - if systemctl is-active --quiet nginx.service; then - echo "gateway leaf rotated — re-importing and reloading nginx" - systemctl restart hive-gateway-self-signed-cert.service - systemctl reload nginx.service - elif [ "$(systemctl show -P LoadState nginx.service)" = loaded ]; then - echo "gateway leaf landed with nginx not running — starting it" - systemctl reset-failed nginx.service - systemctl start --no-block nginx.service - fi - ''; - }; - - # The swarm-services leaf, issued by the secret store's `pki` mount. - # - # The leaf chains straight to the mount's root, with no intermediate: - # the store's `pki/roles/swarm-services` narrows the names with an - # `allowed_domains` list enforced at issue time, so there is no x509 - # `nameConstraints` for an intermediate to carry. - # - # The names it may carry are not this unit's to assert: the role - # refuses anything outside `allowed_domains`, read from the - # swarm-wide `swarm.serviceDomains` that the SANs below are a - # per-host SUBSET of. A mismatch is a refusal from the store naming - # the offending name, not a certificate quietly issued for something - # nobody configured. - systemd.services.swarm-services-cert = { - description = "Issue the swarm-services TLS leaf from the secret store's PKI"; - wantedBy = [ "multi-user.target" ]; - # ⚠️ Where the gateway waits for this leaf, the store being up is a - # precondition of the gateway's certificate, and `requiredBy` is - # what makes a store that never comes up an outage that says so - # instead of a gateway serving the wrong name. - # - # 🩸 NOT `Before=hive-tls-ca.service`, however much the trust bundle - # wants it. That edge closed a cycle — the store's container is - # `After=hive-tls-ca` (it serves the material that unit writes), and - # this unit is `After=` the container — so systemd broke the loop the - # only way it can, by deleting a job: - # - # Found ordering cycle: container@swarm-bao.service/start after - # hive-tls-ca.service/start after swarm-services-cert.service/start - # Job swarm-services-cert.service/start deleted to break ordering cycle - # - # With the job gone the leaf was never issued, `Requires=` took the - # gateway's cert import down with it, and nginx never started at all — - # so the swarm served nothing rather than serving the wrong name. The - # bundle reaches the root the other way round, at the end of the script - # below: it restarts `hive-tls-ca` when the root CHANGED, which is the - # same path that already covered a store coming up hours late. - before = lib.optional gatewayWaitsForServicesLeaf "hive-gateway-self-signed-cert.service"; - requiredBy = lib.optional gatewayWaitsForServicesLeaf "hive-gateway-self-signed-cert.service"; - # The store's container, where it runs here. On a hive that reads a - # store hosted elsewhere no such unit exists and systemd ignores - # the name, which is the correct behaviour rather than a gap: what - # this unit actually needs is the store reachable, and on a remote - # store nothing local can order against that. - # - # `swarm-bao-pki` mints the services-issuer leaf the login below - # presents, and is absent on a host holding an operator-placed one — - # ignored there for the same reason, rather than being a dependency - # only some hosts may declare. - after = [ - "container@${baoCfg.machine}.service" - "swarm-bao-pki.service" - "swarm-bao-services-issuer-policy.service" - ]; - wants = [ "container@${baoCfg.machine}.service" ]; - path = servicesCertPath; - # ./lib/store-retry.nix: the login below fails for as long as the - # store is sealed or down. - inherit (storeRetry) startLimitBurst startLimitIntervalSec; - serviceConfig = storeRetry.serviceConfig // { - Type = "oneshot"; - RemainAfterExit = true; - UMask = "0077"; - SyslogIdentifier = "swarm-services-cert"; - TimeoutStartSec = servicesCertTimeoutStartSec; + chmod 0644 "$bundle" + ''; }; - environment = servicesCertEnvironment; - script = '' - set -euo pipefail - d=${lib.escapeShellArg cfg.stateDir} - install -d -m 0755 "$d" - # `swarm-services-cert-renew` runs this same script, and two runs - # issuing at once can leave one's key beside the other's leaf. The - # second waits and then finds the leaf fresh. - exec 9>"$d/.swarm-services-cert.lock" - if ! flock -w 600 9; then - echo "another swarm-services issuance has held $d/.swarm-services-cert.lock for 10 minutes" >&2 - exit 1 - fi + # The gateway leaf: issued by the secret store's `pki` mount through + # this hive's own role, at boot, on every deploy and from the daily + # timer below, whenever it is missing, past half its lifetime, missing + # a configured name, not chained to the store's root, or not matching + # its key. + # + # ⚠️ Nothing orders the gateway against this unit, which is what lets a + # hive holding a valid leaf start its gateway with the store sealed or + # unreachable. Only a hive with no leaf yet has a gateway that waits + # for the store: its cert import fails until this unit lands a leaf + # and starts nginx. + # + # Failure leaves the current leaf alone: every file the script writes + # is a `.new` until the store has answered with both fields. + # + # Propagation mechanism: nginx serves a *copy* of the leaf, written by + # `hive-gateway-self-signed-cert` at the mode nginx can read. Issuing + # the source therefore changes nothing on its own — the copy has to be + # remade and nginx reloaded, which is what the calls at the end do. + # Neither of the running-nginx calls is `|| true`: a swallowed + # propagation failure means the leaf rotates on disk while nginx keeps + # serving the old copy until it expires, with this unit reporting + # success the whole time. + systemd.services.hive-tls-resign = lib.mkIf runsHive { + description = "Issue or renew the gateway TLS leaf from the secret store's PKI"; + wantedBy = [ "multi-user.target" ]; + # The store's container and its granting unit where they run here; + # ignored elsewhere, as for `swarm-services-cert`, which writes the + # store's root this unit checks the leaf against. + after = [ + "hive-tls-ca.service" + "container@${baoCfg.machine}.service" + "swarm-bao-pki.service" + "swarm-bao-hive-leaf-policy.service" + "swarm-services-cert.service" + ]; + wants = [ "container@${baoCfg.machine}.service" ]; + path = [ + baoDeploy.package + pkgs.jq + pkgs.openssl + pkgs.coreutils + pkgs.systemd + ]; + environment = hiveLeafEnvironment; + # ./lib/store-retry.nix: the login below fails for as long as the + # store is sealed or down. + inherit (storeRetry) startLimitBurst startLimitIntervalSec; + serviceConfig = storeRetry.serviceConfig // { + Type = "oneshot"; + UMask = "0077"; + SyslogIdentifier = "hive-tls-resign"; + # One attempt. A oneshot has no start timeout of its own, and an + # attempt that never ends never fails into a retry. + TimeoutStartSec = "10min"; + }; + script = '' + set -euo pipefail + d=${lib.escapeShellArg cfg.stateDir} + install -d -m 0755 "$d" + leaf="$d/gateway.pem" + leafk="$d/gateway-key.pem" - ${leafCoverage} + fail() { + echo "$1" >&2 + if [ -s "$leaf" ]; then + echo "the current gateway leaf at $leaf stays in place" >&2 + fi + exit 1 + } - # No leaf to ask for, but the root still belongs in the trust - # bundle: every other hive's swarm-service names chain to it. - # `ca/pem` takes no token; the TLS client certificate is the - # store's listener's demand, not bao's, so this presents the - # hive's own identity and never logs in with it. - # A hive without a readable identity fails here, naming the - # options: every hive must hold one. - if [ "$want_svc" = 0 ]; then - ${ - if haveHiveIdentity then - requireHiveIdentity "fetch the store's ${baoServicesPkiMount} root" - + '' - echo "no swarm service domains configured — fetching only the store's pki root" - err="$(mktemp)" - raw="$(mktemp "$d/swarm-services-root.XXXXXX")" - trap 'rm -f "$err" "$raw"' EXIT - if ! curl --fail --silent --show-error \ - --connect-timeout 10 --max-time 30 \ - ${storeRootCurlTls} --output "$raw" \ - "$BAO_ADDR/v1/${baoServicesPkiMount}/ca/pem" 2>"$err"; then - echo "could not read the store's ${baoServicesPkiMount} root." >&2 - cat "$err" >&2 - exit 1 - fi - # Re-encoded rather than copied: it fails on a body that does not - # parse as a certificate, and ends the file in the newline the - # bundle's `cat` relies on. - if ! openssl x509 -in "$raw" -out "$svcroot.new" 2>"$err"; then - echo "the store's ${baoServicesPkiMount} root does not parse as a certificate." >&2 - cat "$err" >&2 - exit 1 - fi - if cmp -s "$svcroot.new" "$svcroot"; then - rm -f "$svcroot.new" - echo "the store's pki root is unchanged" - exit 0 - fi - mv -f "$svcroot.new" "$svcroot" - chmod 0644 "$svcroot" - if systemctl is-active --quiet hive-tls-ca.service; then - echo "the services root changed — rebuilding the trust bundle" - systemctl restart --no-block hive-tls-ca.service - fi - exit 0'' - else - requireHiveIdentity "fetch the store's ${baoServicesPkiMount} root" - } - fi + # Checked before the leaf is: a hive without its identity is red + # even while its current leaf is still valid. + ${ + if hiveLeafRole == null then + ''fail "services.hyperhive.hiveName is unset, so this hive has no gateway-leaf role to issue from."'' + else + requireHiveIdentity "ask the store for its gateway leaf" + } - # Re-issue when the file is missing, past half the lifetime the - # store's role grants it, or no longer carrying every configured - # name. The name check is what makes adding a swarm service take - # effect on the rebuild that added it rather than whenever the - # certificate happens to lapse. + ${leafCoverage} + + # Half the lifetime the role grants, as for the services leaf, so + # the daily timer has half the lifetime to retry through a sealed + # store. + renewat=$(( ${toString baoDeploy.servicesPkiLeafTtlHours} * 3600 / 2 )) + reissue=0 + { [ -s "$leaf" ] && [ -s "$leafk" ]; } || reissue=1 + openssl x509 -in "$leaf" -noout -checkend "$renewat" >/dev/null 2>&1 || reissue=1 + leavesCoverNames || reissue=1 + # A leaf that does not chain to the store's current root, whoever + # signed it, fails this and is re-issued. + { [ -s "$svcroot" ] && openssl verify -CAfile "$svcroot" "$leaf" >/dev/null 2>&1; } || reissue=1 + # The leaf and the key are two renames, so an interrupted rotation can + # leave a certificate beside a key it does not belong to. + leafpub="$(openssl x509 -in "$leaf" -noout -pubkey 2>/dev/null || true)" + keypub="$(openssl pkey -in "$leafk" -pubout 2>/dev/null || true)" + { [ -n "$leafpub" ] && [ "$leafpub" = "$keypub" ]; } || reissue=1 + + if [ "$reissue" = 0 ]; then + echo "gateway leaf valid, matching its key, chained to the store's root and covering the configured names — leaving it alone" + exit 0 + fi + + err="$(mktemp)" + resp="$(mktemp "$d/gateway.json.XXXXXX")" + trap 'rm -f "$err" "$resp" "$leaf.new" "$leafk.new"' EXIT + + # Cert auth is a login, not a transport setting; `-token-only` + # skips the token helper, for the reason `swarm-services-cert` gives. + if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then + fail "could not log in to the swarm secret store with this hive's identity: $(cat "$err")" + fi + export BAO_TOKEN + + echo "requesting the gateway leaf from the store" + if ! bao write -format=json \ + ${lib.escapeShellArg "${baoServicesPkiMount}/issue/${toString hiveLeafRole}"} \ + common_name=${lib.escapeShellArg domain} \ + alt_names=${lib.escapeShellArg (lib.concatStringsSep "," (lib.tail hiveLeafNames))} \ + > "$resp" 2>"$err"; then + fail "the store refused to issue the gateway leaf: $(cat "$err")" + fi + + # One issuance, split, for the reasons `swarm-services-cert` gives: + # the key never enters an argv, and `// empty` keeps a missing field + # from becoming the four characters `null`. The leaf alone, without + # its issuer: the issuer is the self-signed root in the bundle. + umask 077 + jq -r '.data.private_key // empty' < "$resp" > "$leafk.new" + jq -r '.data.certificate // empty' < "$resp" > "$leaf.new" + for f in "$leafk.new" "$leaf.new"; do + [ -s "$f" ] || fail "the store's response was missing a field: $f is empty" + done + + mv -f "$leaf.new" "$leaf" + mv -f "$leafk.new" "$leafk" + chmod 0644 "$leaf" + chmod 0600 "$leafk" + + # A gateway already running gets the copy remade and a reload. One + # that is not running — a hive that had no leaf — gets started, + # `--no-block` so a start job still queued absorbs the request. + if systemctl is-active --quiet nginx.service; then + echo "gateway leaf rotated — re-importing and reloading nginx" + systemctl restart hive-gateway-self-signed-cert.service + systemctl reload nginx.service + elif [ "$(systemctl show -P LoadState nginx.service)" = loaded ]; then + echo "gateway leaf landed with nginx not running — starting it" + systemctl reset-failed nginx.service + systemctl start --no-block nginx.service + fi + ''; + }; + + # The swarm-services leaf, issued by the secret store's `pki` mount. + # + # The leaf chains straight to the mount's root, with no intermediate: + # the store's `pki/roles/swarm-services` narrows the names with an + # `allowed_domains` list enforced at issue time, so there is no x509 + # `nameConstraints` for an intermediate to carry. + # + # The names it may carry are not this unit's to assert: the role + # refuses anything outside `allowed_domains`, read from the + # swarm-wide `swarm.serviceDomains` that the SANs below are a + # per-host SUBSET of. A mismatch is a refusal from the store naming + # the offending name, not a certificate quietly issued for something + # nobody configured. + systemd.services.swarm-services-cert = { + description = "Issue the swarm-services TLS leaf from the secret store's PKI"; + wantedBy = [ "multi-user.target" ]; + # ⚠️ Where the gateway waits for this leaf, the store being up is a + # precondition of the gateway's certificate, and `requiredBy` is + # what makes a store that never comes up an outage that says so + # instead of a gateway serving the wrong name. # - # Half, as `hive-tls-resign` does for the hive leaf, so the daily - # renewal has half the lifetime to retry through a sealed store. - # Not the whole lifetime: a leaf issued with exactly that much left - # is inside it the second after, so every run re-issued. - renewat=$(( ${toString baoDeploy.servicesPkiLeafTtlHours} * 3600 / 2 )) - reissue=0 - { [ -s "$svcleaf" ] && [ -s "$d/swarm-services-key.pem" ]; } || reissue=1 - openssl x509 -in "$svcleaf" -noout -checkend "$renewat" >/dev/null 2>&1 || reissue=1 - covers "$svcleaf" "$svcNames" || reissue=1 - [ -s "$svcroot" ] || reissue=1 - # A leaf that does not chain to the root the trust bundle holds, - # whoever signed it, fails this and is re-issued. - openssl verify -CAfile "$svcroot" "$svcleaf" >/dev/null 2>&1 || reissue=1 - - # The ROOT's window, not the leaf's, and it is what keeps the two - # sides of this from needing to talk. The store replaces an issuer - # that has less than a leaf's window left (./swarm-bao.nix's - # regeneration guard) — so a hive testing its own copy of that same - # certificate against the same threshold, read from the same option, - # asks for a new leaf on the same activation, and gets the - # replacement root back with it. Without this a leaf stays "valid" - # while the anchor it chains to no longer exists in the mount, which - # no expiry check would ever catch. - openssl x509 -in "$svcroot" -noout -checkend ${ - toString (baoDeploy.servicesPkiLeafTtlHours * 3600) - } >/dev/null 2>&1 || reissue=1 - - if [ "$reissue" = 0 ]; then - echo "swarm-services leaf valid and covering the configured names — leaving it alone" - exit 0 - fi - - ${requireServicesIssuer "ask the store for the swarm-services leaf"} - - # `bao`'s own message is the only thing separating an unreachable - # store from a refused identity from a role that would not issue - # these names, and this unit retries on all three — so it reports - # which one rather than asserting all three in a sentence of ours. - err="$(mktemp)" - trap 'rm -f "$err"' EXIT - - # Cert auth is a login, not a transport setting: the - # `BAO_CLIENT_*` variables only decide which certificate the - # handshake presents. Without a token `bao` asks its token - # helper, and that is a `sh` this unit's `path` does not carry — - # `-token-only` answers on stdout and skips the helper. - if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then - echo "could not log in to the swarm secret store with this host's services-issuer certificate." >&2 - cat "$err" >&2 - exit 1 - fi - export BAO_TOKEN - - echo "requesting the swarm-services leaf from the store" - resp="$(mktemp "$d/swarm-services.json.XXXXXX")" - trap 'rm -f "$err" "$resp"' EXIT - - # ONE call, and the response carries all three artefacts — - # certificate, private key and issuing CA. Splitting it into a - # local CSR plus `pki/sign` would buy a key that never crosses - # the wire, and cost the thing that makes this unit worth having: - # the store, not this host, decides what the certificate says. - if ! bao write -format=json \ - ${lib.escapeShellArg "${baoServicesPkiMount}/issue/${baoServicesPkiRole}"} \ - common_name=${lib.escapeShellArg leafCommonName} \ - alt_names=${lib.escapeShellArg (lib.concatStringsSep "," localServiceDomains)} \ - > "$resp" 2>"$err"; then - echo "the store refused to issue the swarm-services certificate." >&2 - cat "$err" >&2 - exit 1 - fi - - # Three fields out of ONE issuance, which is why the response is - # captured and then split rather than asked for three times with - # `bao write -field=…`: each of those calls would mint a separate - # certificate, and the key would belong to a different one than - # the leaf. + # 🩸 NOT `Before=hive-tls-ca.service`, however much the trust bundle + # wants it. That edge closed a cycle — the store's container is + # `After=hive-tls-ca` (it serves the material that unit writes), and + # this unit is `After=` the container — so systemd broke the loop the + # only way it can, by deleting a job: # - # `jq -r` into a file the shell opened, never into a variable and - # never into an argv — the private key does not become visible in - # /proc on its way to disk. + # Found ordering cycle: container@swarm-bao.service/start after + # hive-tls-ca.service/start after swarm-services-cert.service/start + # Job swarm-services-cert.service/start deleted to break ordering cycle # - # `// empty` on every one of them: `jq -r` renders a missing - # field as the four characters `null`, which is a non-empty file - # and would sail past the check below into a certificate slot. - umask 077 - jq -r '.data.private_key // empty' < "$resp" > "$d/swarm-services-key.pem.new" - jq -r '.data.certificate // empty' < "$resp" > "$d/swarm-services-leaf.new" - jq -r '.data.issuing_ca // empty' < "$resp" > "$svcroot.new" + # With the job gone the leaf was never issued, `Requires=` took the + # gateway's cert import down with it, and nginx never started at all — + # so the swarm served nothing rather than serving the wrong name. The + # bundle reaches the root the other way round, at the end of the script + # below: it restarts `hive-tls-ca` when the root CHANGED, which is the + # same path that already covered a store coming up hours late. + before = lib.optional gatewayWaitsForServicesLeaf "hive-gateway-self-signed-cert.service"; + requiredBy = lib.optional gatewayWaitsForServicesLeaf "hive-gateway-self-signed-cert.service"; + # The store's container, where it runs here. On a hive that reads a + # store hosted elsewhere no such unit exists and systemd ignores + # the name, which is the correct behaviour rather than a gap: what + # this unit actually needs is the store reachable, and on a remote + # store nothing local can order against that. + # + # `swarm-bao-pki` mints the services-issuer leaf the login below + # presents, and is absent on a host holding an operator-placed one — + # ignored there for the same reason, rather than being a dependency + # only some hosts may declare. + after = [ + "container@${baoCfg.machine}.service" + "swarm-bao-pki.service" + "swarm-bao-services-issuer-policy.service" + ]; + wants = [ "container@${baoCfg.machine}.service" ]; + path = servicesCertPath; + # ./lib/store-retry.nix: the login below fails for as long as the + # store is sealed or down. + inherit (storeRetry) startLimitBurst startLimitIntervalSec; + serviceConfig = storeRetry.serviceConfig // { + Type = "oneshot"; + RemainAfterExit = true; + UMask = "0077"; + SyslogIdentifier = "swarm-services-cert"; + TimeoutStartSec = servicesCertTimeoutStartSec; + }; + environment = servicesCertEnvironment; + script = '' + set -euo pipefail + d=${lib.escapeShellArg cfg.stateDir} + install -d -m 0755 "$d" - for f in "$d/swarm-services-key.pem.new" "$d/swarm-services-leaf.new" "$svcroot.new"; do - if [ ! -s "$f" ]; then - echo "the store's response was missing a field: $f is empty" >&2 + # `swarm-services-cert-renew` runs this same script, and two runs + # issuing at once can leave one's key beside the other's leaf. The + # second waits and then finds the leaf fresh. + exec 9>"$d/.swarm-services-cert.lock" + if ! flock -w 600 9; then + echo "another swarm-services issuance has held $d/.swarm-services-cert.lock for 10 minutes" >&2 exit 1 fi - done - # nginx serves this file verbatim. It carries the leaf ALONE and - # not leaf-plus-issuer, which is the one visible consequence of - # flattening the chain: the issuer here is a self-signed root, so - # appending it would ship an anchor a client either already holds - # — in which case it is redundant — or does not, in which case - # shipping it proves nothing. - mv -f "$d/swarm-services-leaf.new" "$svcleaf" - mv -f "$d/swarm-services-key.pem.new" "$d/swarm-services-key.pem" - chmod 0644 "$svcleaf" - chmod 0600 "$d/swarm-services-key.pem" + ${leafCoverage} - # The anchor, world-readable, beside the leaf it signed. This is - # the file an operator adds to a browser's trust store, and the - # one `hive-tls-ca` folds into `trust-bundle.pem`. Public - # certificate material only — the private half never leaves the - # store's `pki` mount, which is what `root/generate/internal` - # buys. - rootchanged=0 - if ! cmp -s "$svcroot.new" "$svcroot"; then - rootchanged=1 - fi - mv -f "$svcroot.new" "$svcroot" - chmod 0644 "$svcroot" + # No leaf to ask for, but the root still belongs in the trust + # bundle: every other hive's swarm-service names chain to it. + # `ca/pem` takes no token; the TLS client certificate is the + # store's listener's demand, not bao's, so this presents the + # hive's own identity and never logs in with it. + # A hive without a readable identity fails here, naming the + # options: every hive must hold one. + if [ "$want_svc" = 0 ]; then + ${ + if haveHiveIdentity then + requireHiveIdentity "fetch the store's ${baoServicesPkiMount} root" + + '' + echo "no swarm service domains configured — fetching only the store's pki root" + err="$(mktemp)" + raw="$(mktemp "$d/swarm-services-root.XXXXXX")" + trap 'rm -f "$err" "$raw"' EXIT + if ! curl --fail --silent --show-error \ + --connect-timeout 10 --max-time 30 \ + ${storeRootCurlTls} --output "$raw" \ + "$BAO_ADDR/v1/${baoServicesPkiMount}/ca/pem" 2>"$err"; then + echo "could not read the store's ${baoServicesPkiMount} root." >&2 + cat "$err" >&2 + exit 1 + fi + # Re-encoded rather than copied: it fails on a body that does not + # parse as a certificate, and ends the file in the newline the + # bundle's `cat` relies on. + if ! openssl x509 -in "$raw" -out "$svcroot.new" 2>"$err"; then + echo "the store's ${baoServicesPkiMount} root does not parse as a certificate." >&2 + cat "$err" >&2 + exit 1 + fi + if cmp -s "$svcroot.new" "$svcroot"; then + rm -f "$svcroot.new" + echo "the store's pki root is unchanged" + exit 0 + fi + mv -f "$svcroot.new" "$svcroot" + chmod 0644 "$svcroot" + if systemctl is-active --quiet hive-tls-ca.service; then + echo "the services root changed — rebuilding the trust bundle" + systemctl restart --no-block hive-tls-ca.service + fi + exit 0'' + else + requireHiveIdentity "fetch the store's ${baoServicesPkiMount} root" + } + fi - # Propagation. nginx serves a *copy* of the leaf, so re-issuing the - # source changes nothing until the copy is remade. A running - # gateway (`swarm-services-cert-renew` rotating the leaf) gets the - # copy remade and a reload. A gateway that is not running gets - # started: a start job still waiting on this unit absorbs the - # request, and a start that already ended on this unit's - # `requiredBy` has nothing else that would start it again. - # - # ⚠️ `--no-block`, and it is not a preference. This unit declares - # `Before=` both of these, so a blocking `systemctl restart` or - # `start` enqueues a job that systemd will not start until this - # unit is active — and this unit is not active until its ExecStart - # returns, which is waiting on that job. A deadlock, broken only - # when `TimeoutStartSec` fails the attempt, and entered again by - # every retry. Queueing the job and letting it run once we exit is - # the only ordering that terminates. - # - # The cost is real and is the reason `hive-tls-resign` does NOT - # do this: a propagation that fails after we return does not fail - # us, so the leaf can rotate on disk while nginx keeps the old - # copy. That unit blocks because nothing orders it before its - # consumers; this one cannot. - if systemctl is-active --quiet nginx.service; then - echo "swarm-services leaf rotated — re-importing and reloading nginx" - systemctl restart --no-block hive-gateway-self-signed-cert.service - systemctl reload --no-block nginx.service - elif [ "$(systemctl show -P LoadState nginx.service)" = loaded ]; then - echo "swarm-services leaf landed with nginx not running — starting it" - systemctl reset-failed nginx.service - systemctl start --no-block nginx.service - fi + # Re-issue when the file is missing, past half the lifetime the + # store's role grants it, or no longer carrying every configured + # name. The name check is what makes adding a swarm service take + # effect on the rebuild that added it rather than whenever the + # certificate happens to lapse. + # + # Half, as `hive-tls-resign` does for the hive leaf, so the daily + # renewal has half the lifetime to retry through a sealed store. + # Not the whole lifetime: a leaf issued with exactly that much left + # is inside it the second after, so every run re-issued. + renewat=$(( ${toString baoDeploy.servicesPkiLeafTtlHours} * 3600 / 2 )) + reissue=0 + { [ -s "$svcleaf" ] && [ -s "$d/swarm-services-key.pem" ]; } || reissue=1 + openssl x509 -in "$svcleaf" -noout -checkend "$renewat" >/dev/null 2>&1 || reissue=1 + covers "$svcleaf" "$svcNames" || reissue=1 + [ -s "$svcroot" ] || reissue=1 + # A leaf that does not chain to the root the trust bundle holds, + # whoever signed it, fails this and is re-issued. + openssl verify -CAfile "$svcroot" "$svcleaf" >/dev/null 2>&1 || reissue=1 - # The bundle is assembled by `hive-tls-ca`, which runs BEFORE this - # unit — it has to, or the store's container cannot start and there - # is nothing to ask for a leaf. So on a first boot it wrote a bundle - # without the root below, and this restart is what completes it - # rather than a retry-path fallback. Only on a CHANGED root: every - # boot otherwise bounces a unit with nothing to do. - if [ "$rootchanged" = 1 ] && systemctl is-active --quiet hive-tls-ca.service; then - echo "the services root changed — rebuilding the trust bundle" - systemctl restart --no-block hive-tls-ca.service - fi - ''; - }; + # The ROOT's window, not the leaf's, and it is what keeps the two + # sides of this from needing to talk. The store replaces an issuer + # that has less than a leaf's window left (./swarm-bao.nix's + # regeneration guard) — so a hive testing its own copy of that same + # certificate against the same threshold, read from the same option, + # asks for a new leaf on the same activation, and gets the + # replacement root back with it. Without this a leaf stays "valid" + # while the anchor it chains to no longer exists in the mount, which + # no expiry check would ever catch. + openssl x509 -in "$svcroot" -noout -checkend ${ + toString (baoDeploy.servicesPkiLeafTtlHours * 3600) + } >/dev/null 2>&1 || reissue=1 - # The services leaf's renewal: `swarm-services-cert`'s own script, run - # again from the daily timer below. - # - # ⚠️ A unit of its own, not a timer on `swarm-services-cert`. That unit - # is `RemainAfterExit`, so once it has run it stays active and a timer - # starting it does nothing. And it cannot be restarted instead: - # `hive-gateway-self-signed-cert` `Requires=` it, so a restart restarts - # the gateway's cert import and nginx behind it, and a store that is - # sealed at that moment fails the start and takes both down over a leaf - # that was still valid. Nothing requires or orders against this unit, so - # its failure is its own. - # - # Failure leaves the leaf alone: every file the script writes is a - # `.new` until the store has answered with all three fields. No - # `Restart=`: the unit stays failed until the next tick, which is how - # a sealed store shows up in `systemctl --failed`. - systemd.services.swarm-services-cert-renew = { - description = "Renew the swarm-services TLS leaf from the secret store's PKI"; - # The timer's `Persistent=` can fire it during boot. Behind the boot - # issuance, so it finds the leaf that run just wrote. - after = [ "swarm-services-cert.service" ]; - path = servicesCertPath; - environment = servicesCertEnvironment; - inherit (config.systemd.services.swarm-services-cert) script; - serviceConfig = { - Type = "oneshot"; - UMask = "0077"; - SyslogIdentifier = "swarm-services-cert-renew"; - TimeoutStartSec = servicesCertTimeoutStartSec; + if [ "$reissue" = 0 ]; then + echo "swarm-services leaf valid and covering the configured names — leaving it alone" + exit 0 + fi + + ${requireServicesIssuer "ask the store for the swarm-services leaf"} + + # `bao`'s own message is the only thing separating an unreachable + # store from a refused identity from a role that would not issue + # these names, and this unit retries on all three — so it reports + # which one rather than asserting all three in a sentence of ours. + err="$(mktemp)" + trap 'rm -f "$err"' EXIT + + # Cert auth is a login, not a transport setting: the + # `BAO_CLIENT_*` variables only decide which certificate the + # handshake presents. Without a token `bao` asks its token + # helper, and that is a `sh` this unit's `path` does not carry — + # `-token-only` answers on stdout and skips the helper. + if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then + echo "could not log in to the swarm secret store with this host's services-issuer certificate." >&2 + cat "$err" >&2 + exit 1 + fi + export BAO_TOKEN + + echo "requesting the swarm-services leaf from the store" + resp="$(mktemp "$d/swarm-services.json.XXXXXX")" + trap 'rm -f "$err" "$resp"' EXIT + + # ONE call, and the response carries all three artefacts — + # certificate, private key and issuing CA. Splitting it into a + # local CSR plus `pki/sign` would buy a key that never crosses + # the wire, and cost the thing that makes this unit worth having: + # the store, not this host, decides what the certificate says. + if ! bao write -format=json \ + ${lib.escapeShellArg "${baoServicesPkiMount}/issue/${baoServicesPkiRole}"} \ + common_name=${lib.escapeShellArg leafCommonName} \ + alt_names=${lib.escapeShellArg (lib.concatStringsSep "," localServiceDomains)} \ + > "$resp" 2>"$err"; then + echo "the store refused to issue the swarm-services certificate." >&2 + cat "$err" >&2 + exit 1 + fi + + # Three fields out of ONE issuance, which is why the response is + # captured and then split rather than asked for three times with + # `bao write -field=…`: each of those calls would mint a separate + # certificate, and the key would belong to a different one than + # the leaf. + # + # `jq -r` into a file the shell opened, never into a variable and + # never into an argv — the private key does not become visible in + # /proc on its way to disk. + # + # `// empty` on every one of them: `jq -r` renders a missing + # field as the four characters `null`, which is a non-empty file + # and would sail past the check below into a certificate slot. + umask 077 + jq -r '.data.private_key // empty' < "$resp" > "$d/swarm-services-key.pem.new" + jq -r '.data.certificate // empty' < "$resp" > "$d/swarm-services-leaf.new" + jq -r '.data.issuing_ca // empty' < "$resp" > "$svcroot.new" + + for f in "$d/swarm-services-key.pem.new" "$d/swarm-services-leaf.new" "$svcroot.new"; do + if [ ! -s "$f" ]; then + echo "the store's response was missing a field: $f is empty" >&2 + exit 1 + fi + done + + # nginx serves this file verbatim. It carries the leaf ALONE and + # not leaf-plus-issuer, which is the one visible consequence of + # flattening the chain: the issuer here is a self-signed root, so + # appending it would ship an anchor a client either already holds + # — in which case it is redundant — or does not, in which case + # shipping it proves nothing. + mv -f "$d/swarm-services-leaf.new" "$svcleaf" + mv -f "$d/swarm-services-key.pem.new" "$d/swarm-services-key.pem" + chmod 0644 "$svcleaf" + chmod 0600 "$d/swarm-services-key.pem" + + # The anchor, world-readable, beside the leaf it signed. This is + # the file an operator adds to a browser's trust store, and the + # one `hive-tls-ca` folds into `trust-bundle.pem`. Public + # certificate material only — the private half never leaves the + # store's `pki` mount, which is what `root/generate/internal` + # buys. + rootchanged=0 + if ! cmp -s "$svcroot.new" "$svcroot"; then + rootchanged=1 + fi + mv -f "$svcroot.new" "$svcroot" + chmod 0644 "$svcroot" + + # Propagation. nginx serves a *copy* of the leaf, so re-issuing the + # source changes nothing until the copy is remade. A running + # gateway (`swarm-services-cert-renew` rotating the leaf) gets the + # copy remade and a reload. A gateway that is not running gets + # started: a start job still waiting on this unit absorbs the + # request, and a start that already ended on this unit's + # `requiredBy` has nothing else that would start it again. + # + # ⚠️ `--no-block`, and it is not a preference. This unit declares + # `Before=` both of these, so a blocking `systemctl restart` or + # `start` enqueues a job that systemd will not start until this + # unit is active — and this unit is not active until its ExecStart + # returns, which is waiting on that job. A deadlock, broken only + # when `TimeoutStartSec` fails the attempt, and entered again by + # every retry. Queueing the job and letting it run once we exit is + # the only ordering that terminates. + # + # The cost is real and is the reason `hive-tls-resign` does NOT + # do this: a propagation that fails after we return does not fail + # us, so the leaf can rotate on disk while nginx keeps the old + # copy. That unit blocks because nothing orders it before its + # consumers; this one cannot. + if systemctl is-active --quiet nginx.service; then + echo "swarm-services leaf rotated — re-importing and reloading nginx" + systemctl restart --no-block hive-gateway-self-signed-cert.service + systemctl reload --no-block nginx.service + elif [ "$(systemctl show -P LoadState nginx.service)" = loaded ]; then + echo "swarm-services leaf landed with nginx not running — starting it" + systemctl reset-failed nginx.service + systemctl start --no-block nginx.service + fi + + # The bundle is assembled by `hive-tls-ca`, which runs BEFORE this + # unit — it has to, or the store's container cannot start and there + # is nothing to ask for a leaf. So on a first boot it wrote a bundle + # without the root below, and this restart is what completes it + # rather than a retry-path fallback. Only on a CHANGED root: every + # boot otherwise bounces a unit with nothing to do. + if [ "$rootchanged" = 1 ] && systemctl is-active --quiet hive-tls-ca.service; then + echo "the services root changed — rebuilding the trust bundle" + systemctl restart --no-block hive-tls-ca.service + fi + ''; }; - }; - # Daily, for the reason `swarm-services-cert-renew` below is: the leaf - # needs the store, which can be sealed or unreachable. - systemd.timers.hive-tls-resign = lib.mkIf runsHive { - description = "Daily renewal check for the gateway TLS leaf"; - wantedBy = [ "timers.target" ]; - timerConfig = { - OnCalendar = "daily"; - Persistent = true; + # The services leaf's renewal: `swarm-services-cert`'s own script, run + # again from the daily timer below. + # + # ⚠️ A unit of its own, not a timer on `swarm-services-cert`. That unit + # is `RemainAfterExit`, so once it has run it stays active and a timer + # starting it does nothing. And it cannot be restarted instead: + # `hive-gateway-self-signed-cert` `Requires=` it, so a restart restarts + # the gateway's cert import and nginx behind it, and a store that is + # sealed at that moment fails the start and takes both down over a leaf + # that was still valid. Nothing requires or orders against this unit, so + # its failure is its own. + # + # Failure leaves the leaf alone: every file the script writes is a + # `.new` until the store has answered with all three fields. No + # `Restart=`: the unit stays failed until the next tick, which is how + # a sealed store shows up in `systemctl --failed`. + systemd.services.swarm-services-cert-renew = { + description = "Renew the swarm-services TLS leaf from the secret store's PKI"; + # The timer's `Persistent=` can fire it during boot. Behind the boot + # issuance, so it finds the leaf that run just wrote. + after = [ "swarm-services-cert.service" ]; + path = servicesCertPath; + environment = servicesCertEnvironment; + inherit (config.systemd.services.swarm-services-cert) script; + serviceConfig = { + Type = "oneshot"; + UMask = "0077"; + SyslogIdentifier = "swarm-services-cert-renew"; + TimeoutStartSec = servicesCertTimeoutStartSec; + }; }; - }; - # Daily rather than weekly, for the reason - # ./swarm-nats.nix's `swarm-bao-nats-tls` timer is: this leaf needs the - # store, which can be sealed or unreachable, so there is a retry per day - # from half-life to expiry rather than two. - systemd.timers.swarm-services-cert-renew = { - description = "Daily renewal check for the swarm-services TLS leaf"; - wantedBy = [ "timers.target" ]; - timerConfig = { - OnCalendar = "daily"; - Persistent = true; + # Daily, for the reason `swarm-services-cert-renew` below is: the leaf + # needs the store, which can be sealed or unreachable. + systemd.timers.hive-tls-resign = lib.mkIf runsHive { + description = "Daily renewal check for the gateway TLS leaf"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = "daily"; + Persistent = true; + }; }; - }; - }; + + # Daily rather than weekly, for the reason + # ./swarm-nats.nix's `swarm-bao-nats-tls` timer is: this leaf needs the + # store, which can be sealed or unreachable, so there is a retry per day + # from half-life to expiry rather than two. + systemd.timers.swarm-services-cert-renew = { + description = "Daily renewal check for the swarm-services TLS leaf"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = "daily"; + Persistent = true; + }; + }; + }) + ]; } diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 198d7d41..aa2626e6 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -2470,7 +2470,7 @@ in every gateway host, whether or not a hive runs there. Required on every self-signed gateway host that fronts a swarm-service - name ({option}`services.hyperhive.swarm.localServiceDomains` not + name ({option}`services.hyperhive.deploy.hive-controller.tls.localServiceDomains` not empty). Unset there, the host evaluates and `swarm-services-cert` fails, naming both options. A host fronting none only fetches the store's root, presenting diff --git a/nix/host-modules/swarm.nix b/nix/host-modules/swarm.nix index 6ff9aa62..84608bea 100644 --- a/nix/host-modules/swarm.nix +++ b/nix/host-modules/swarm.nix @@ -90,7 +90,7 @@ let # Every name is listed whether or not this host runs the service: the # store's issuing role is built from this list on the store's host and # has to admit a leaf for any gateway in the swarm. What one gateway's - # leaf carries is `localServiceDomains`. + # leaf carries is `deploy.hive-controller.tls.localServiceDomains`. # # Membership is also what `gateway.lib.tlsFor` consults to pick the # services leaf over the hive one, so a gateway-published swarm name @@ -278,27 +278,6 @@ in ''; }; - options.services.hyperhive.swarm.localServiceDomains = lib.mkOption { - type = lib.types.listOf lib.types.str; - readOnly = true; - internal = true; - description = '' - Read-only: the names in `serviceDomains` **this host actually - fronts**, which is what its services leaf may carry. The swarm-wide - set stays what the sub-CA is name-constrained to and what the - store's issuing role permits; narrowing the leaf is what stops a - private key on one gateway presenting a valid certificate for - services that host does not serve and never has. - - Read off the rendered `services.nginx.virtualHosts` rather than off - the per-service deploy flags: those flags are what the vhosts are - already guarded on, so a second reading of them is a copy that - drifts from the thing it claims to describe. Filtering - `serviceDomains` rather than assembling a list beside it is that - option's own constraint, and it applies here for the same reason. - ''; - }; - config = { # Total on a null `swarm.domain` for the reason `hives..domain`'s # default gives. @@ -317,10 +296,6 @@ in lib.unique (lib.filter (d: d != null && d != "") serviceDomains') ); - services.hyperhive.swarm.localServiceDomains = lib.filter ( - d: config.services.nginx.virtualHosts ? ${d} - ) swarmCfg.serviceDomains; - assertions = [ { # An EMPTY `hives` fires this too, deliberately: since diff --git a/nix/module-eval/bao-basics.nix b/nix/module-eval/bao-basics.nix index c4614e67..1ee24246 100644 --- a/nix/module-eval/bao-basics.nix +++ b/nix/module-eval/bao-basics.nix @@ -374,7 +374,7 @@ let m: builtins.elem "hive-gateway-self-signed-cert.service" m.systemd.services.swarm-services-cert.before; in - baoPkcs11.services.hyperhive.swarm.localServiceDomains == [ "bao-ui.t.local" ] + baoPkcs11.services.hyperhive.deploy.hive-controller.tls.localServiceDomains == [ "bao-ui.t.local" ] && baoPkcs11.systemd.services ? hive-gateway-self-signed-cert && !(waits baoPkcs11) && !(orders baoPkcs11) diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index 0f6adda0..fe53eb4a 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -361,7 +361,7 @@ let # vhosts are the hive leaf's, which this host still signs. name = "a host fronting none of the swarm's service names requests no services leaf"; ok = - bare.services.hyperhive.swarm.localServiceDomains == [ ] + bare.services.hyperhive.deploy.hive-controller.tls.localServiceDomains == [ ] && bare.services.hyperhive.swarm.serviceDomains != [ ] && lib.hasInfix "want_svc=0" bare.systemd.services.swarm-services-cert.script; } diff --git a/nix/module-eval/hive-tls.nix b/nix/module-eval/hive-tls.nix index ffda8b80..abab4d62 100644 --- a/nix/module-eval/hive-tls.nix +++ b/nix/module-eval/hive-tls.nix @@ -222,7 +222,7 @@ let # that asks the store for no leaf. name = "a hive fronting no swarm name puts the store's pki root in its trust bundle"; ok = - noService.services.hyperhive.swarm.localServiceDomains == [ ] + noService.services.hyperhive.deploy.hive-controller.tls.localServiceDomains == [ ] && lib.hasInfix ''"$BAO_ADDR/v1/pki/ca/pem"'' noSvcFetch && lib.hasInfix ''mv -f "$svcroot.new" "$svcroot"'' noSvcFetch && lib.hasInfix ''svcroot="$d/swarm-services-root.pem"'' noService.systemd.services.hive-tls-ca.script @@ -255,7 +255,7 @@ let # control is the same hive with an identity, whose branch fetches. name = "a hive fronting no swarm name with no store identity fails the unit, naming the hive identity's options"; ok = - noIdentity.services.hyperhive.swarm.localServiceDomains == [ ] + noIdentity.services.hyperhive.deploy.hive-controller.tls.localServiceDomains == [ ] && noIdentity.services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile == null && !(lib.hasInfix "/v1/pki/ca/pem" noIdFetch) && lib.hasInfix "baoHiveIdentityCertFile (unset)" (noSvcBranch noIdFetch) diff --git a/nix/module-eval/nats-tls.nix b/nix/module-eval/nats-tls.nix index bc983e76..2ced25bf 100644 --- a/nix/module-eval/nats-tls.nix +++ b/nix/module-eval/nats-tls.nix @@ -420,7 +420,7 @@ let && queueNoStore.services.nginx.virtualHosts.${natsName}.sslCertificate == "/var/lib/swarm-nats-gateway-tls/cert.pem" - && h.swarm.localServiceDomains == [ ] + && h.deploy.hive-controller.tls.localServiceDomains == [ ] && !(lib.elem natsName h.swarm.serviceDomains) && !(lib.elem "hive-gateway-self-signed-cert.service" (s.swarm-services-cert.requiredBy or [ ])) && !(lib.hasInfix "serves swarm names only with it" s.hive-gateway-self-signed-cert.script) diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix index 0bf2343a..4f5acde7 100644 --- a/nix/module-eval/swarm-services-switch.nix +++ b/nix/module-eval/swarm-services-switch.nix @@ -500,7 +500,7 @@ let # The store's UI alone: its nginx carries the passthrough to the # store, so it starts on the fallback with the store sealed. && - store.services.hyperhive.swarm.localServiceDomains + store.services.hyperhive.deploy.hive-controller.tls.localServiceDomains == [ store.services.hyperhive.swarm.bao.ui.domain ] && !(waitsForLeaf store) # The homeserver without a hive: no hive-domain vhost at all, and the