diff --git a/docs/swarm/ca.md b/docs/swarm/ca.md index 2a770671..c527a1df 100644 --- a/docs/swarm/ca.md +++ b/docs/swarm/ca.md @@ -74,6 +74,17 @@ re-asserts the role and the grant without touching the anchor. A root that changed per boot would invalidate every certificate issued under it and every browser taught to trust it. +**A daily timer renews the leaf.** The store's role issues it for 720 +hours. `swarm-services-cert-renew.timer` runs the same script as +`swarm-services-cert.service` once a day, and that script asks the store +for a new leaf once the current one is past half that window or is +missing a configured name. A rotation re-imports the leaf into the +gateway and reloads nginx. A store that can't answer — still sealed, or +out of reach — fails the unit, which leaves the current leaf in place, +and the next day's run tries again: about fifteen tries before the leaf +lapses. `systemctl +list-timers swarm-services-cert-renew` shows the next run. + ## Constraints on the material The root's private key never reaches the nix store: the store is diff --git a/nix/checks.nix b/nix/checks.nix index 547ac621..80ed4874 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -43,6 +43,10 @@ in inherit pkgs self nixosSystem; inherit (pkgs) lib; }; + module-eval-hive-tls = import ./module-eval/hive-tls.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; module-eval-matrix-core = import ./module-eval/matrix-core.nix { inherit pkgs self nixosSystem; inherit (pkgs) lib; diff --git a/nix/host-modules/hive-tls.nix b/nix/host-modules/hive-tls.nix index 7771b705..77064e98 100644 --- a/nix/host-modules/hive-tls.nix +++ b/nix/host-modules/hive-tls.nix @@ -247,6 +247,40 @@ let covers "$leaf" "$hiveNames" } ''; + + # What `swarm-services-cert` and `swarm-services-cert-renew` both run the + # issuance with. Bound here rather than read back off the first unit: its + # merged `path` already carries systemd's defaults, so the second would + # render a different `PATH` and fail to merge. + servicesCertPath = [ + baoDeploy.package + pkgs.jq + pkgs.openssl + pkgs.coreutils + pkgs.systemd + # `cmp`, which is NOT in coreutils. Without it the root-changed test + # below is `command not found` — 127, invisible under `if !`, and + # therefore always "changed", so the bundle rebuild it guards fired + # on every issuance instead of on a new root. + pkgs.diffutils + # `flock`, for the lock at the top of the script. + pkgs.util-linux + ]; + servicesCertEnvironment = { + BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; + } + // lib.optionalAttrs (cfg.baoClientCertFile != null) { + BAO_CLIENT_CERT = cfg.baoClientCertFile; + } + // lib.optionalAttrs (cfg.baoClientKeyFile != null) { + BAO_CLIENT_KEY = cfg.baoClientKeyFile; + } + # Absent means the system trust store, which is what a deployment + # with a real CA wants and what a self-signed one must not be left + # with. + // lib.optionalAttrs (baoDeploy.serverCaFile != null) { + BAO_CACERT = baoDeploy.serverCaFile; + }; in { # Host-side TLS trust root for the self-signed gateway mode. @@ -367,11 +401,14 @@ in # rebuild of every hive that is not the CA host, about a fallback # that no longer happens. - # Both run on the deploy and sit on the gateway's start path, so a - # failure in either takes TLS down for every service behind it. + # The first two run on the deploy and sit on the gateway's start path, + # so a failure in either takes TLS down for every service behind it. + # The renewal sits on no start path, and a failure there is a leaf that + # lapses days later unless someone reads why it failed now. services.hyperhive.swarm.otel.journaldUnits = [ "hive-tls-ca" "swarm-services-cert" + "swarm-services-cert-renew" ]; # Generate (and rotate) the hive CA + gateway leaf before anything @@ -598,8 +635,8 @@ in # # The services leaf is not one of them any longer: it is issued # by the secret store, whose key this unit does not hold and - # cannot re-sign under. Renewing it is `swarm-services-cert`'s - # job, at boot, which is a cadence its own issue owns. + # cannot re-sign under. `swarm-services-cert-renew` renews it, + # on its own daily timer. ${leafCoverage} # Re-sign only when a leaf is within half its validity of expiry. @@ -694,18 +731,7 @@ in "swarm-bao-services-issuer-policy.service" ]; wants = [ "container@${baoCfg.machine}.service" ]; - path = [ - baoDeploy.package - pkgs.jq - pkgs.openssl - pkgs.coreutils - pkgs.systemd - # `cmp`, which is NOT in coreutils. Without it the root-changed test - # below is `command not found` — 127, invisible under `if !`, and - # therefore always "changed", so the bundle rebuild it guards fired - # on every issuance instead of on a new root. - pkgs.diffutils - ]; + path = servicesCertPath; # Sized like the store's own granting units, and for the same # reason: under `seal = "shamir"` an operator unseals BY HAND, and # the login below fails for as long as that takes. 2880 × 30s is @@ -722,26 +748,21 @@ in Restart = "on-failure"; RestartSec = 30; }; - environment = { - BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; - } - // lib.optionalAttrs (cfg.baoClientCertFile != null) { - BAO_CLIENT_CERT = cfg.baoClientCertFile; - } - // lib.optionalAttrs (cfg.baoClientKeyFile != null) { - BAO_CLIENT_KEY = cfg.baoClientKeyFile; - } - # Absent means the system trust store, which is what a deployment - # with a real CA wants and what a self-signed one must not be left - # with. - // lib.optionalAttrs (baoDeploy.serverCaFile != null) { - BAO_CACERT = baoDeploy.serverCaFile; - }; + environment = servicesCertEnvironment; script = '' set -euo pipefail d=${lib.escapeShellArg cfg.stateDir} install -d -m 0755 "$d" + # `swarm-services-cert-renew` runs this same script, and two runs + # issuing at once can leave one's key beside the other's leaf. The + # second waits and then finds the leaf fresh. + exec 9>"$d/.swarm-services-cert.lock" + if ! flock -w 600 9; then + echo "another swarm-services issuance has held $d/.swarm-services-cert.lock for 10 minutes" >&2 + exit 1 + fi + ${leafCoverage} if [ "$want_svc" = 0 ]; then @@ -749,14 +770,20 @@ in exit 0 fi - # Same rule as the hive leaf's, at a different cadence: re-issue - # when the file is missing, within 30 days of expiry, or no - # longer carrying every configured name. The name check is what - # makes adding a swarm service take effect on the rebuild that - # added it rather than whenever the certificate happens to lapse. + # Re-issue when the file is missing, past half the lifetime the + # store's role grants it, or no longer carrying every configured + # name. The name check is what makes adding a swarm service take + # effect on the rebuild that added it rather than whenever the + # certificate happens to lapse. + # + # Half, as `hive-tls-resign` does for the hive leaf, so the daily + # renewal has half the lifetime to retry through a sealed store. + # Not the whole lifetime: a leaf issued with exactly that much left + # is inside it the second after, so every run re-issued. + renewat=$(( ${toString baoDeploy.servicesPkiLeafTtlHours} * 3600 / 2 )) reissue=0 { [ -s "$svcleaf" ] && [ -s "$d/swarm-services-key.pem" ]; } || reissue=1 - openssl x509 -in "$svcleaf" -noout -checkend 2592000 >/dev/null 2>&1 || reissue=1 + openssl x509 -in "$svcleaf" -noout -checkend "$renewat" >/dev/null 2>&1 || reissue=1 covers "$svcleaf" "$svcNames" || reissue=1 [ -s "$svcroot" ] || reissue=1 @@ -764,11 +791,14 @@ in # sides of this from needing to talk. The store replaces an issuer # that has less than a leaf's window left (./swarm-bao.nix's # regeneration guard) — so a hive testing its own copy of that same - # certificate against the same threshold asks for a new leaf on the - # same activation, and gets the replacement root back with it. - # Without this a leaf stays "valid" while the anchor it chains to no - # longer exists in the mount, which no expiry check would ever catch. - openssl x509 -in "$svcroot" -noout -checkend 2592000 >/dev/null 2>&1 || reissue=1 + # certificate against the same threshold, read from the same option, + # asks for a new leaf on the same activation, and gets the + # replacement root back with it. Without this a leaf stays "valid" + # while the anchor it chains to no longer exists in the mount, which + # no expiry check would ever catch. + openssl x509 -in "$svcroot" -noout -checkend ${ + toString (baoDeploy.servicesPkiLeafTtlHours * 3600) + } >/dev/null 2>&1 || reissue=1 if [ "$reissue" = 0 ]; then echo "swarm-services leaf valid and covering the configured names — leaving it alone" @@ -877,12 +907,13 @@ in mv -f "$svcroot.new" "$svcroot" chmod 0644 "$svcroot" - # Propagation, for the RETRY path only. Ordered before both of - # these, so on a normal boot they have not run yet, `is-active` - # is false, and ordering alone does the work. What this covers is - # the store coming up hours after the gateway did: nginx serves a - # *copy* of the leaf, so re-issuing the source changes nothing - # until the copy is remade. + # Propagation, for the retry and renewal paths only. Ordered before + # both of these, so on a normal boot they have not run yet, + # `is-active` is false, and ordering alone does the work. What this + # covers is the store coming up hours after the gateway did, and + # `swarm-services-cert-renew` rotating the leaf under a running + # gateway: nginx serves a *copy* of the leaf, so re-issuing the + # source changes nothing until the copy is remade. # # ⚠️ `--no-block`, and it is not a preference. This unit declares # `Before=` both of these, so a blocking `systemctl restart` @@ -917,6 +948,37 @@ in ''; }; + # The services leaf's renewal: `swarm-services-cert`'s own script, run + # again from the daily timer below. + # + # ⚠️ A unit of its own, not a timer on `swarm-services-cert`. That unit + # is `RemainAfterExit`, so once it has run it stays active and a timer + # starting it does nothing. And it cannot be restarted instead: + # `hive-gateway-self-signed-cert` `Requires=` it, so a restart restarts + # the gateway's cert import and nginx behind it, and a store that is + # sealed at that moment fails the start and takes both down over a leaf + # that was still valid. Nothing requires or orders against this unit, so + # its failure is its own. + # + # Failure leaves the leaf alone: every file the script writes is a + # `.new` until the store has answered with all three fields. No + # `Restart=`: the unit stays failed until the next tick, which is how + # a sealed store shows up in `systemctl --failed`. + systemd.services.swarm-services-cert-renew = { + description = "Renew the swarm-services TLS leaf from the secret store's PKI"; + # The timer's `Persistent=` can fire it during boot. Behind the boot + # issuance, so it finds the leaf that run just wrote. + after = [ "swarm-services-cert.service" ]; + path = servicesCertPath; + environment = servicesCertEnvironment; + inherit (config.systemd.services.swarm-services-cert) script; + serviceConfig = { + Type = "oneshot"; + UMask = "0077"; + SyslogIdentifier = "swarm-services-cert-renew"; + }; + }; + systemd.timers.hive-tls-resign = { description = "Weekly gateway-leaf re-sign and propagation"; wantedBy = [ "timers.target" ]; @@ -929,6 +991,19 @@ in }; }; + # Daily rather than `hive-tls-resign`'s weekly, for the reason + # ./swarm-nats.nix's `swarm-bao-nats-tls` timer is: this leaf needs the + # store, which can be sealed or unreachable, so there is a retry per day + # from half-life to expiry rather than two. + systemd.timers.swarm-services-cert-renew = { + description = "Daily renewal check for the swarm-services TLS leaf"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = "daily"; + Persistent = true; + }; + }; + # Signal the hive-c0re lifecycle that a hive CA exists: it bind-mounts # this file (read-only, public certs ONLY — never a key) into each # agent container so agents + their tools can trust the gateway's diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 372c088a..44404842 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -376,8 +376,9 @@ let # compares the issuer's remaining life against this number, which it can # only do if this number exists. 720h matches # `deploy.hive-controller.tls.leafValidityDays`, the 30 days ./hive-tls.nix - # documents for every other leaf it holds. - servicesPkiLeafTtlHours = 720; + # documents for every other leaf it holds. An option, read here and by + # ./hive-tls.nix's renewal threshold, so the two cannot disagree. + servicesPkiLeafTtlHours = baoDeploy.servicesPkiLeafTtlHours; servicesPkiLeafTtl = "${toString servicesPkiLeafTtlHours}h"; servicesPkiLeafTtlSeconds = servicesPkiLeafTtlHours * 3600; @@ -1107,6 +1108,20 @@ in ''; }; + servicesPkiLeafTtlHours = lib.mkOption { + type = lib.types.ints.positive; + internal = true; + default = 720; + description = '' + Lifetime, in hours, of a leaf issued through + {option}`services.hyperhive.deploy.bao.servicesPkiRoleName`: the role's + `ttl` and `max_ttl`. ./hive-tls.nix renews the swarm-services leaf at + half of it, and a threshold spelled as its own number there drifts the + day this one changes. Internal rather than read-only so a check can + move it and see the threshold move too. + ''; + }; + servicesIssuerCommonName = lib.mkOption { type = lib.types.str; default = "swarm-services-issuer"; diff --git a/nix/module-eval/hive-tls.nix b/nix/module-eval/hive-tls.nix new file mode 100644 index 00000000..31f60a9f --- /dev/null +++ b/nix/module-eval/hive-tls.nix @@ -0,0 +1,128 @@ +# `checks.module-eval-hive-tls` — see ./lib.nix for the shared rationale (why +# this suite exists, naming convention, "evaluates not executes"). +# +# The swarm-services leaf's renewal: a timer that really re-runs the +# issuance, at a threshold read off the store role's lifetime. +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + hive + runGroup + ; + + # Every service on one host, with a bootstrap token so the store's granting + # unit renders the role this leaf is issued through. + allLocal = hive { + deploy.singleHostSwarm = true; + deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; + }; + + # The same host with the role's lifetime moved, so a threshold that is a + # number of its own shows up as one that did not move with it. + shortTtl = hive { + deploy.singleHostSwarm = true; + deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; + deploy.bao.servicesPkiLeafTtlHours = 48; + }; + + units = allLocal.systemd.services; + boot = units.swarm-services-cert; + timer = allLocal.systemd.timers.swarm-services-cert-renew; + + # What the timer starts, read off the timer rather than assumed from its + # name: a timer's `Unit=` defaults to its own name, and pointing it at the + # boot unit is exactly the regression the cases below exist for. + triggeredName = lib.removeSuffix ".service" ( + timer.timerConfig.Unit or "swarm-services-cert-renew.service" + ); + triggered = units.${triggeredName}; + + remainsAfterExit = u: u.serviceConfig.RemainAfterExit or false; + + renewAt = hours: "renewat=$(( ${toString hours} * 3600 / 2 ))"; + + cases = [ + { + # Control: the boot issuance is the unit a timer cannot re-run. Were it + # not `RemainAfterExit`, the case after next would pass vacuously. + name = "the boot issuance renders, and stays active after it exits"; + ok = lib.hasInfix "pki/issue/swarm-services" boot.script && remainsAfterExit boot; + } + { + name = "a timer for the services leaf is enabled and fires daily"; + ok = + lib.elem "timers.target" timer.wantedBy + && timer.timerConfig.OnCalendar == "daily" + && timer.timerConfig.Persistent; + } + { + # Starting an active unit is a no-op, so a timer aimed at a + # `RemainAfterExit` unit fires on schedule and renews nothing. + name = "the timer starts a unit that re-runs the issuance and does not stay active"; + ok = + units ? ${triggeredName} + && !(remainsAfterExit triggered) + && triggered.script == boot.script + # The whole set, `PATH` included: an environment copied off the + # boot unit's merged options renders a second `PATH` and fails. + && triggered.environment == boot.environment; + } + { + # A restart of anything the gateway's cert import `Requires=` takes + # nginx down with it, so the renewal must be something nothing else + # depends on, and must run behind the boot issuance. + name = "nothing requires or waits on the renewal, and it runs after the boot issuance"; + ok = + (triggered.requiredBy or [ ]) == [ ] + && (triggered.wantedBy or [ ]) == [ ] + && (triggered.before or [ ]) == [ ] + && lib.elem "swarm-services-cert.service" triggered.after + && !(lib.any (u: lib.elem "${triggeredName}.service" ((u.requires or [ ]) ++ (u.after or [ ]))) ( + lib.attrValues (removeAttrs units [ triggeredName ]) + )); + } + { + name = "the renewal's journal ships beside the boot issuance's"; + ok = lib.elem triggeredName allLocal.services.hyperhive.swarm.otel.journaldUnits; + } + { + # Moved with the role, in both places: the threshold is half of what the + # store grants, and the store grants what the option says. + name = "the leaf is renewed at half the role's lifetime, read from the option"; + ok = + lib.hasInfix (renewAt 720) boot.script + && lib.hasInfix (renewAt 48) shortTtl.systemd.services.swarm-services-cert.script + && lib.hasInfix ''-in "$svcleaf" -noout -checkend "$renewat"'' boot.script + && lib.hasInfix "max_ttl=48h" shortTtl.systemd.services.swarm-bao-controller-policy.script; + } + { + # The store replaces its root once it has less than one leaf lifetime + # left, and the hive asks for a fresh leaf, with the new root, at that + # same threshold. A number of its own here keeps a leaf whose root the + # store has already replaced. + name = "the services root is re-checked at the store's own replacement threshold"; + ok = + let + shortBoot = shortTtl.systemd.services.swarm-services-cert.script; + shortStore = shortTtl.systemd.services.swarm-bao-controller-policy.script; + in + lib.hasInfix ''-in "$svcroot" -noout -checkend ${toString (720 * 3600)} '' boot.script + && lib.hasInfix ''-in "$svcroot" -noout -checkend ${toString (48 * 3600)} '' shortBoot + && lib.hasInfix "-checkend ${toString (48 * 3600)} <<<\"$root_ca\"" shortStore; + } + ]; +in +runGroup "hive-tls" cases