hyperhive/nix/host-modules/hive-tls.nix

{
  lib,
  config,
  pkgs,
  ...
}:
let
  cfg = config.services.hyperhive.deploy.hive-controller.tls;
  hyperhiveCfg = config.services.hyperhive;
  gatewayCfg = config.services.hyperhive.gateway;
  swarmCaCfg = config.services.hyperhive.swarm.ca;
  domain = hyperhiveCfg.domain;
  baoCfg = hyperhiveCfg.swarm.bao;
  baoDeploy = hyperhiveCfg.deploy.bao;

  # The endpoint the services leaf comes out of. Read off the store's own
  # options rather than spelled here: ./swarm-bao.nix enables the mount,
  # generates its root and writes the grant that names this exact path, and a
  # second module composing its own is how the grant and the caller stop
  # agreeing.
  baoServicesPkiMount = baoDeploy.servicesPkiMountPath;
  baoServicesPkiRole = baoDeploy.servicesPkiRoleName;

  # Derived once in ./swarm.nix and read here + in ./swarm-bao.nix, so
  # the names this leaf carries as SANs and the names the store's
  # `pki/roles/swarm-services` is narrowed to cannot disagree.
  swarmServiceDomains = hyperhiveCfg.swarm.serviceDomains;

  # The host-managed hive CA is the trust anchor for self-signed mode.
  # It is only stood up when the gateway actually serves a self-signed
  # cert: the gateway must be in self-signed mode. `domain` is required
  # (asserted in hive-network.nix), so the leaf SANs always have a
  # domain to derive from. The self-signed condition is the gateway
  # module's single source of truth (`gateway.useSelfSigned`): true when
  # neither an operator cert (`tls.certDir`) nor ACME is set.
  active = hyperhiveCfg.enable && gatewayCfg.useSelfSigned;

  # How this hive's CA comes into existence when it is missing — and it
  # is one of exactly two things, chosen by config rather than by what
  # happens to be on disk.
  #
  # Issuing a sub-CA requires the swarm root's PRIVATE key, so it can
  # only happen where that key legitimately lives: the single-host
  # deployment that `swarm.ca.autoConfigure` describes. A host cannot
  # infer that it is that host — a swarm's services and its hives can
  # sit anywhere — so with the flag off this hive self-signs exactly as
  # it always has, and an operator who wants it in the hierarchy
  # installs the CA themselves. Falling back to self-signed rather than
  # failing keeps a plain hive working out of the box; what it loses is
  # membership of a swarm's trust, which is the correct thing to lose
  # for a hive nobody has federated.
  caGenScript =
    if swarmCaCfg.autoConfigure then
      ''
        # The root should exist — `swarm-ca.service` runs before this and
        # is required by it. If it doesn't, something upstream failed and
        # signing with a half-provisioned root would be worse than stopping.
        if [ ! -s "$root" ] || [ ! -s "$rootk" ]; then
          echo "swarm.ca.autoConfigure is set but there is no root CA key at $rootk" >&2
          echo "(swarm-ca.service should have generated it) — refusing to issue a hive CA." >&2
          exit 1
        fi

        echo "issuing fresh hive CA at $ca under the swarm root"
        cacsr="$(mktemp "$d/ca.csr.XXXXXX")"
        caext="$(mktemp "$d/ca.ext.XXXXXX")"
        trap 'rm -f "$cacsr" "$caext"' EXIT

        openssl req -newkey rsa:4096 -nodes -sha256 \
          -keyout "$cak" -out "$cacsr" \
          -subj "/CN=hive-ca ${domain}"

        # printf (not a heredoc) so the ext-file lines carry no leading
        # whitespace once nix has stripped the indented-string indent.
        {
          printf 'basicConstraints=critical,CA:TRUE,pathlen:0\n'
          printf 'keyUsage=critical,keyCertSign,cRLSign\n'
          printf 'subjectKeyIdentifier=hash\n'
          printf 'authorityKeyIdentifier=keyid:always\n'
          # The constraint is the point of the hierarchy, not a
          # flourish: without it a leaked hive CA mints any name in
          # the swarm, and it is verifiers that enforce this, not our
          # good behaviour. The IP exclusions are not redundant — a
          # DNS constraint says nothing about an iPAddress SAN, and a
          # name type nobody constrained is a name type this CA is
          # unconstrained for.
          printf 'nameConstraints=critical,permitted;DNS:%s,excluded;IP:0.0.0.0/0.0.0.0,excluded;IP:0:0:0:0:0:0:0:0/0:0:0:0:0:0:0:0\n' \
            ${lib.escapeShellArg domain}
        } > "$caext"

        openssl x509 -req -in "$cacsr" -CA "$root" -CAkey "$rootk" \
          -CAcreateserial -days ${toString cfg.caValidityDays} -sha256 \
          -extfile "$caext" -out "$ca"
        rm -f "$cacsr" "$caext"
        trap - EXIT
      ''
    else
      ''
        echo "generating fresh self-signed hive CA at $ca"
        openssl req -x509 -newkey rsa:4096 -nodes -sha256 \
          -days ${toString cfg.caValidityDays} \
          -keyout "$cak" -out "$ca" \
          -subj "/CN=hive-ca ${domain}" \
          -addext "basicConstraints=critical,CA:TRUE,pathlen:0" \
          -addext "keyUsage=critical,keyCertSign,cRLSign"
      '';

  # Sign one leaf. Parameterised rather than hardcoded to `gateway.*`
  # because there are now two: the hive's own leaf, issued by the hive
  # CA, and the swarm-services leaf, issued by the services sub-CA that
  # ./swarm-ca.nix maintains. Same ceremony, different issuer and names
  # — and one script means the two cannot drift in how they are built.
  #
  #   $1 stateDir  $2 basename  $3 CN  $4 SAN list  $5 issuer cert  $6 issuer key
  signLeafScript = pkgs.writeShellScript "hive-tls-sign-leaf" ''
    set -euo pipefail
    d="$1"
    base="$2"
    cn="$3"
    sans="$4"
    ca="$5"
    cak="$6"
    leaf="$d/$base.pem"
    leafk="$d/$base-key.pem"
    csr="$(mktemp "$d/$base.csr.XXXXXX")"
    ext="$(mktemp "$d/$base.ext.XXXXXX")"
    only="$(mktemp "$d/$base.leaf.XXXXXX")"
    trap 'rm -f "$csr" "$ext" "$only"' EXIT

    openssl req -newkey rsa:4096 -nodes -sha256 \
      -keyout "$leafk" -out "$csr" \
      -subj "/CN=$cn"

    # printf (not a heredoc) so the ext-file lines carry no leading
    # whitespace once nix has stripped the indented-string indent.
    {
      printf 'subjectAltName=%s\n' "$sans"
      printf 'basicConstraints=critical,CA:FALSE\n'
      printf 'keyUsage=critical,digitalSignature,keyEncipherment\n'
      printf 'extendedKeyUsage=serverAuth\n'
    } > "$ext"

    openssl x509 -req -in "$csr" -CA "$ca" -CAkey "$cak" \
      -CAcreateserial -days ${toString cfg.leafValidityDays} -sha256 \
      -extfile "$ext" -out "$only"

    # nginx serves this file verbatim, so it must carry the leaf AND its
    # issuer: the hive CA is an intermediate under the swarm root, and a
    # client that anchors on the root cannot build the middle of the
    # chain by itself. Agents anchor on the hive CA directly and
    # validated either way — the appended cert is what makes a swarm
    # peer, or anything else holding only the root, work.
    # Leaf first: both `ssl_certificate` and the `openssl x509 -in`
    # expiry checks read the first cert in the file.
    cat "$only" "$ca" > "$leaf"
    chmod 0600 "$leafk"
    chmod 0644 "$leaf"
  '';

  # The hive's own leaf: signed by the hive CA, covering the hive domain
  # and its sub-domains.
  #
  # `forge.` and `matrix.` were listed here explicitly and both were
  # redundant with the wildcard beside them — a wildcard covers exactly
  # one label, and those are one label. Naming them read as policy
  # ("these two services live under this leaf"), which is the reason they
  # outlived it: a configured service name is no longer necessarily under
  # this domain, and when it isn't, this is the one list it cannot join.
  # The hive CA is `nameConstraints=permitted;DNS:<hive domain>`, and a
  # violation invalidates the CERTIFICATE rather than the offending name
  # — measured with openssl, not reasoned about — so one sibling name
  # added here takes the leaf down for the dashboard and every other
  # vhost sharing it. Configured service names get the swarm-services
  # leaf below, whose sub-CA is constrained to exactly those names.
  signHiveLeaf = ''
    ${signLeafScript} "$d" gateway ${lib.escapeShellArg domain} \
      ${lib.escapeShellArg "DNS:${domain},DNS:*.${domain}"} \
      "$d/ca.pem" "$d/ca-key.pem"
  '';

  # Shared by every unit that decides whether to re-issue a leaf. It
  # lives here rather than in one of them because the guards have to
  # agree: they answer the same question at different times
  # (`hive-tls-ca` at service activation, i.e. on the deploy;
  # `hive-tls-resign` from a weekly timer for a host that stays up long
  # enough to drift; `swarm-services-cert` for the leaf the store
  # issues). A rule implemented in one and not the others is worse than
  # one implemented in none — it looks fixed and only fires on whichever
  # path you did not take, which is exactly how a corrected
  # `serviceDomains` still served a stale leaf.
  #
  # Expects `$d` (state dir) to be set; defines `$leaf`-adjacent names and
  # `covers`.
  leafCoverage = ''
    svcleaf="$d/swarm-services.pem"
    svcroot="$d/swarm-services-root.pem"
    hiveNames=${lib.escapeShellArg "${domain} *.${domain}"}
    svcNames=${lib.escapeShellArg (lib.concatStringsSep " " swarmServiceDomains)}

    # A hive with no configured service names has no services leaf to
    # hold, and its absence there is the correct state rather than a
    # stale one. Anywhere else it is expected: the leaf comes from the
    # secret store's `pki` mount now, not from a sub-CA that exists on
    # one host in the swarm, so "this host cannot issue it" is no longer
    # one of the answers.
    want_svc=${if swarmServiceDomains == [ ] then "0" else "1"}

    # Expiry is not the only way a leaf goes wrong. One signed when the
    # configured name set was smaller stays valid for its whole lifetime
    # while omitting every name added since — so a config change can
    # evaluate, build and deploy cleanly while the gateway keeps serving a
    # certificate that does not cover the new service.
    #
    # Reads the names out of the CERTIFICATE, not a sidecar file: the pem
    # is what nginx serves, and a bookkeeping file drifts from it the
    # moment anyone replaces a leaf by hand.
    covers() { # $1 = leaf, $2 = space-separated names it must carry
      [ -s "$1" ] || return 1
      _have="$(openssl x509 -in "$1" -noout -ext subjectAltName 2>/dev/null \
               | tr ',' '\n' | sed -n 's/^[[:space:]]*DNS:\(.*\)$/\1/p' || true)"
      [ -n "$_have" ] || return 1
      _missing=0
      # ⚠️ The hive leaf carries `*.<domain>`; without this the shell
      # expands it against the cwd and the comparison silently tests a
      # filename instead of a name.
      set -f
      for _n in $2; do
        printf '%s\n' "$_have" | grep -qxF "$_n" || _missing=1
      done
      set +f
      [ "$_missing" = 0 ]
    }

    # True when the leaf this host SIGNS is present and carries its
    # configured names. Expiry is the callers' own business — they use
    # different windows.
    #
    # ⚠️ The services leaf is deliberately NOT part of this any more, and
    # the omission is load-bearing rather than a simplification. Its two
    # callers sign under the hive CA's key; the services leaf comes out
    # of the secret store, which neither of them can reach. A guard that
    # went on reporting it stale would make `hive-tls-ca` re-sign the
    # hive leaf forever over a file it has no way to fix. The
    # `swarm-services-cert` unit below holds the same rule for that leaf,
    # using the `covers` helper above so the two cannot drift.
    leavesCoverNames() {
      covers "$leaf" "$hiveNames"
    }
  '';
in
{
  # Host-side TLS trust root for the self-signed gateway mode.
  #
  # A bare self-signed leaf would be its own trust anchor, so every
  # regeneration would be a new anchor and every consumer (agents,
  # federation peers) would have to re-trust on each rotation — and a
  # runtime-generated, in-container cert can't be wired into an agent's
  # build-time trust store at all.
  #
  # So the issuer is a long-lived **hive CA** held on the host. The
  # gateway serves a **leaf** signed by that CA (via the `tls.certDir`
  # bind-mount path); agents and federation peers trust the *CA* once,
  # and leaf rotation never re-breaks them. See `docs/networking/gateway.md`
  # ("Self-signed TLS").
  #
  # That CA is self-signed by default. Under
  # `swarm.ca.autoConfigure` — the all-on-one-host case — it is instead
  # an intermediate issued under the swarm root (./swarm-ca.nix owns the
  # why) and name-constrained to this hive's domain, so it stays the
  # anchor agents pin while a peer holding only the root can validate
  # everything this hive serves. Either way an existing CA is left
  # alone; see the issuance comment below.

  options.services.hyperhive.deploy.hive-controller.tls = {
    stateDir = lib.mkOption {
      type = lib.types.str;
      default = "/var/lib/hive-tls";
      description = ''
        Host directory holding the hive CA + gateway leaf cert for the
        self-signed gateway mode. `ca.pem` (the anchor agents and
        federation peers trust), `ca-key.pem` (0600, never leaves the
        host), `gateway.pem` / `gateway-key.pem` (the leaf the gateway
        container bind-mounts and nginx serves). Persistent so the CA
        survives reboots — re-deriving it would re-break every consumer.
      '';
    };

    caValidityDays = lib.mkOption {
      type = lib.types.int;
      default = 7300;
      description = ''
        Validity window of the hive CA in days (default ~20y). Kept long
        and well beyond `leafValidityDays` so the CA outlives many leaf
        rotations — the whole point of the CA is to be a stable anchor
        that consumers trust once. The CA is regenerated only if missing
        or already expired.
      '';
    };

    baoClientCertFile = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default = null;
      example = "/var/lib/swarm-bao-pki/services-issuer.pem";
      description = ''
        Certificate `swarm-services-cert` presents to the secret store when it
        asks for the swarm-services leaf the gateway serves.

        Its subject has to be
        {option}`services.hyperhive.deploy.bao.servicesIssuerCommonName` — cert
        auth matches on the common name, and this is the one role whose policy
        permits `pki/issue/swarm-services`.

        A **separate** credential from
        {option}`services.hyperhive.deploy.bao.clientCertFile`, which is this
        hive's own reader identity and whose policy reads every secret the
        hive is entitled to. A unit that renews one server certificate has no
        business holding that.

        On a hive that mints its own store PKI a glue module supplies the leaf
        as a `mkDefault`; everywhere else it is the credential an operator
        places, for the reason the store's own client certificate is.

        A path, never a value.
      '';
    };

    baoClientKeyFile = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default = null;
      example = "/var/lib/swarm-bao-pki/services-issuer-key.pem";
      description = ''
        Private key for
        {option}`services.hyperhive.deploy.hive-controller.tls.baoClientCertFile`.
        Both or neither — a certificate with no key authenticates nothing.
      '';
    };

    leafValidityDays = lib.mkOption {
      type = lib.types.int;
      default = 30;
      description = ''
        Validity window of the gateway leaf cert in days (default 30).
        Short-lived by design — ahead of the CA/Browser-Forum's move
        toward ~47-day max lifetimes — which bounds the blast radius of a
        leaf-key compromise. The leaf is re-signed by the (stable) CA
        when it is missing or near expiry; because it shares the CA
        anchor, a rotation does not disturb consumer trust. Agents and
        federation peers validate against the CA, not browser CA/B-forum
        limits. The weekly `hive-tls-resign` timer re-signs the leaf once
        it is within half its validity of expiry and propagates the new
        leaf into the running gateway, so a long-uptime host renews
        automatically without a reboot.
      '';
    };
  };

  config = lib.mkIf active {
    # 🚫 The "these service names are outside this hive's domain and this
    # host cannot sign for them" warning is GONE, and its absence is the
    # deliverable rather than a tidy-up. It fired on
    # `!swarm.ca.autoConfigure` — i.e. "this host does not hold the swarm
    # root key" — which was the whole reason a hive could end up serving
    # its own leaf on a swarm-service name. The services leaf now comes
    # from the secret store's `pki` mount, which every hive reaches over
    # the network with its own identity, so holding the root key stopped
    # being the thing that decides. Left in place it would warn, on every
    # rebuild of every hive that is not the CA host, about a fallback
    # that no longer happens.

    # Generate (and rotate) the hive CA + gateway leaf before anything
    # serves it. Idempotent: the CA is created once and reused; the leaf
    # is re-signed on expiry under the same CA so the anchor is stable.
    systemd.services.hive-tls-ca = {
      description = "Generate hive CA + gateway leaf TLS cert (self-signed mode)";
      wantedBy = [ "multi-user.target" ];
      # The consumer is `hive-gateway-self-signed-cert`, which copies the
      # leaf into the gateway's state dir at the mode nginx can read, and
      # which nginx in turn `Requires=`. So this must run first or that
      # copy fails under `set -eu` and takes nginx down with it — order
      # against the unit that reads the file.
      before = [ "hive-gateway-self-signed-cert.service" ];
      requiredBy = [ "hive-gateway-self-signed-cert.service" ];
      # The issuance below needs the swarm root key on disk. When this
      # host generates it (single-host swarm) that unit must have run
      # first; when the operator provides the material there is no unit
      # to wait for, so the dependency is conditional rather than a unit
      # that exists and does nothing.
      #
      # `swarm-services-cert` is ordered before this one rather than the
      # other way round, and not because this unit needs the leaf — it
      # never touches it. The trust bundle written at the end of this
      # script anchors on every root a consumer must hold, and the
      # services root is one of them, so the file has to be on disk
      # before the `cat`. Get it wrong and the bundle is a boot behind:
      # correct-looking, and missing the anchor for exactly the names the
      # gateway serves with it.
      after = lib.optionals swarmCaCfg.autoConfigure [ "swarm-ca.service" ];
      requires = lib.optionals swarmCaCfg.autoConfigure [ "swarm-ca.service" ];
      path = [ pkgs.openssl ];
      serviceConfig = {
        Type = "oneshot";
        RemainAfterExit = true;
        UMask = "0077";
        # Pin the journal identity (else it's the `script` store-path wrapper).
        SyslogIdentifier = "hive-tls-ca";
      };
      script = ''
        set -euo pipefail
        d=${lib.escapeShellArg cfg.stateDir}
        install -d -m 0755 "$d"

        ca="$d/ca.pem"
        cak="$d/ca-key.pem"
        leaf="$d/gateway.pem"
        leafk="$d/gateway-key.pem"
        root=${lib.escapeShellArg "${swarmCaCfg.stateDir}/root.pem"}
        rootk=${lib.escapeShellArg "${swarmCaCfg.stateDir}/root-key.pem"}

        prev="$d/ca-previous.pem"
        marker="$d/.swarm-ca-adopted"

        # --- Adoption: a hive whose CA predates the swarm root.
        #
        # Runs ONLY where this host also owns the root (`autoConfigure`),
        # because adoption invalidates an anchor that consumers already
        # trust and they refresh on their own schedule. On one box that
        # schedule is knowable; across hosts it is not, so there the
        # operator does it and this unit only says so, loudly.
        #
        # Guarded by a marker rather than by the state of the world: the
        # marker's ABSENCE is the trigger, so this fires once per hive
        # instead of re-deciding every activation. That is the difference
        # between a migration and a kicking machine.
        if [ -s "$root" ] && [ -s "$ca" ] && [ ! -e "$marker" ] \
            && ! openssl verify -CAfile "$root" "$ca" >/dev/null 2>&1; then
        ${
          if swarmCaCfg.autoConfigure then
            ''
              echo "adopting the swarm CA: $ca does not chain to $root" >&2
              # Keep the old CA as an anchor across the overlap. Consumers
              # read the bundle, so adoption is additive before it is
              # subtractive — agents pick up new trust only when their
              # container restarts, which is a window even on one host.
              cp "$ca" "$prev"
              chmod 0644 "$prev"
              rm -f "$ca" "$cak"
              touch "$marker"''
          else
            ''
              echo "hive-tls: this hive's CA does not chain to the swarm root." >&2
              echo "  hive CA:    $ca" >&2
              echo "  swarm root: $root" >&2
              echo "Adopting the hierarchy is not automatic here: it invalidates an" >&2
              echo "anchor that peers and agents on OTHER hosts still trust, and they" >&2
              echo "refresh on their own schedule — only you know when that is safe." >&2
              echo "To adopt: rm $ca $cak && systemctl restart hive-tls-ca.service" >&2
              echo "To keep the current CA deliberately: touch $marker" >&2
              exit 1''
        }
        fi

        # --- CA: generated once and reused across leaf rotations, in
        # whichever of the two shapes `caGenScript` selected.
        # Regenerated only if missing or already expired (checkend 0).
        # A new CA means every consumer must re-trust, so the leaf is
        # dropped to force a re-sign under the fresh CA.
        #
        # An existing CA is never re-rooted here. A hive predating the
        # swarm root carries a self-signed `ca.pem`, and swapping it for
        # a swarm-issued one would break every consumer already trusting
        # it — including agents, whose trust is refreshed only when their
        # container restarts. Adopting the hierarchy on such a hive is
        # therefore an operator step (drop the CA, let this unit
        # re-issue), which is also what keeps this change non-disruptive
        # to hives that never adopt it.
        if [ ! -s "$ca" ] || [ ! -s "$cak" ] \
            || ! openssl x509 -in "$ca" -noout -checkend 0 >/dev/null 2>&1; then
          ${caGenScript}
          chmod 0600 "$cak"
          chmod 0644 "$ca"
          rm -f "$leaf" "$leafk"
        fi

        # --- Leaf: (re)sign when missing, within 30 days of expiry, or no
        # longer covering the configured names, always under the current
        # (stable) CA.
        ${leafCoverage}

        # ⚠️ This is the guard that runs ON THE DEPLOY — `hive-tls-resign`
        # only fires from a weekly timer, so a rule enforced only there is
        # up to a week late and does nothing for the rebuild that changed
        # the names in the first place.
        #
        resign=0
        { [ -s "$leaf" ] && [ -s "$leafk" ]; } || resign=1
        openssl x509 -in "$leaf" -noout -checkend 2592000 >/dev/null 2>&1 || resign=1
        leavesCoverNames || resign=1

        if [ "$resign" = 1 ]; then
          echo "signing gateway leaf at $leaf (missing, near expiry, or missing a configured name)"
          ${signHiveLeaf}
        fi

        # --- Trust bundle: what a consumer must TRUST, as opposed to
        # `ca.pem`, which is what this host SIGNS with. Why they stopped
        # being the same file, and why `ca-previous.pem` stays in the set
        # after an adoption, are in docs/swarm/ca.md. Three constraints
        # the code cannot state:
        #
        # Written IN PLACE, never renamed into position — containers
        # bind-mount this file and a bind mount follows the inode, so a
        # rename leaves every consumer holding the old one.
        #
        # Dropping `ca-previous.pem` is deliberately NOT done here:
        # "long enough" is a deployment fact, not a unit's call.
        #
        # Built as an array with `if` guards, not
        # `[ -s f ] && anchors+=(f)`: under `set -e` an && list whose test
        # fails IS a failing command and kills the unit — on exactly the
        # hive where the optional file is legitimately absent. The
        # reflexive `|| true` is worse; it also swallows a real failure to
        # read the hive CA.
        #
        # `swarm-services-root.pem` joins the set for the reason the swarm
        # root is in it: it is an ANCHOR a consumer terminates at, not an
        # intermediate. It is also what keeps this change from being a
        # flag day — the swarm root stays in the bundle beside it, so a
        # leaf still chaining to the old services sub-CA and one issued by
        # the store both verify against the same file, and a hive can be
        # rebuilt before or after its peers.
        bundle="$d/trust-bundle.pem"
        anchors=("$ca")
        if [ -s "$prev" ]; then anchors+=("$prev"); fi
        if [ -s "$root" ]; then anchors+=("$root"); fi
        if [ -s "$svcroot" ]; then anchors+=("$svcroot"); fi
        cat "''${anchors[@]}" > "$bundle"
        chmod 0644 "$bundle"
      '';
    };

    # Weekly re-sign of the gateway leaf so short-lived leaves renew
    # without depending on a reboot.
    #
    # `hive-tls-ca` only re-signs at service activation (boot/rebuild); a
    # long-uptime host would otherwise let a 30-day leaf lapse silently.
    # This service re-signs the leaf directly (not by bouncing hive-tls-ca)
    # and propagates the new leaf to nginx when the file actually changed.
    #
    # Propagation mechanism: nginx serves a *copy* of the leaf, written by
    # `hive-gateway-self-signed-cert` at the mode nginx can read. Re-signing
    # the source therefore changes nothing on its own — the copy has to be
    # remade and nginx reloaded, which is what the two calls below do.
    #
    # ⚠️ Neither call is `|| true`: a swallowed propagation failure means
    # the leaf rotates on disk while nginx keeps serving the old copy
    # until it expires, with this unit reporting success the whole time.
    # A failure here must fail the timer.
    systemd.services.hive-tls-resign = {
      description = "Re-sign the gateway TLS leaf and reload nginx";
      # hive-tls-ca must have run first so the CA key exists before we try
      # to re-sign under it. On first boot `Persistent=true` on the weekly
      # timer fires immediately; without this ordering the resign could race
      # the CA initialisation and fail with "no such file" on the CA key.
      after = [ "hive-tls-ca.service" ];
      path = [
        pkgs.openssl
        pkgs.coreutils
        pkgs.systemd
      ];
      serviceConfig = {
        Type = "oneshot";
        UMask = "0077";
        SyslogIdentifier = "hive-tls-resign";
      };
      script = ''
        set -euo pipefail
        d=${lib.escapeShellArg cfg.stateDir}
        leaf="$d/gateway.pem"

        # ⚠️ EVERY leaf this host SIGNS must be covered here. A leaf that
        # first-boot issuance creates and this unit does not know about
        # looks perfect for its entire validity and then expires with no
        # warning — the failure is invisible until it is total. The name
        # checks come from the shared snippet, so this unit and
        # `hive-tls-ca` cannot disagree about what a good leaf is.
        #
        # The services leaf is not one of them any longer: it is issued
        # by the secret store, whose key this unit does not hold and
        # cannot re-sign under. Renewing it is `swarm-services-cert`'s
        # job, at boot, which is a cadence its own issue owns.
        ${leafCoverage}

        # Re-sign only when a leaf is within half its validity of expiry.
        # The weekly cadence catches this window well before one lapses.
        halflife=$(( ${toString cfg.leafValidityDays} * 86400 / 2 ))

        fresh() { # a leaf is fresh if it exists and is not near expiry
          [ -s "$1" ] && openssl x509 -in "$1" -noout -checkend "$halflife" >/dev/null 2>&1
        }

        if fresh "$leaf" && leavesCoverNames; then
          echo "leaves valid, and covering the configured names — no resign needed"
          exit 0
        fi

        echo "a leaf is missing, near expiry, or missing a configured name — re-signing"
        before="$(sha256sum "$leaf" 2>/dev/null || true)"

        ${signHiveLeaf}

        after="$(sha256sum "$leaf" 2>/dev/null || true)"
        if [ "$before" != "$after" ]; then
          echo "gateway leaf rotated — re-importing and reloading nginx"
          systemctl restart hive-gateway-self-signed-cert.service
          systemctl reload nginx.service
        else
          echo "gateway leaf unchanged (already up to date)"
        fi
      '';
    };

    # The swarm-services leaf, issued by the secret store's `pki` mount.
    #
    # ⚠️ This unit replaces a chain, not a command. What used to happen
    # was two openssl hops on whichever host held the swarm root key: a
    # name-constrained `swarm-services` sub-CA (../swarm-ca.nix), then a
    # leaf signed under it here. Both are gone. The store's
    # `pki/roles/swarm-services` carries the same narrowing as an
    # `allowed_domains` list enforced at issue time, so the intermediate
    # that used to encode it in x509 `nameConstraints` has nothing left
    # to express — and with the root inside the mount there is nothing
    # for it to be an intermediate OF.
    #
    # The names it may carry are not this unit's to assert: the role
    # refuses anything outside `allowed_domains`, which is read from the
    # same `swarm.serviceDomains` the SANs below are built from. A
    # mismatch is a refusal from the store naming the offending name,
    # not a certificate quietly issued for something nobody configured.
    systemd.services.swarm-services-cert = {
      description = "Issue the swarm-services TLS leaf from the secret store's PKI";
      wantedBy = [ "multi-user.target" ];
      # ⚠️ The ordering the old shape got by ACCIDENT, written down.
      # Nothing used to declare "no TLS until the store is up" — it held
      # because `swarm-services-ca.service` happened to block the gateway.
      # Now it is a real edge: the gateway's certificate comes out of the
      # store, so the store being up is a precondition rather than a
      # coincidence, and `requiredBy` is what makes a store that never
      # comes up an outage that says so instead of a gateway serving the
      # wrong name.
      before = [
        "hive-tls-ca.service"
        "hive-gateway-self-signed-cert.service"
      ];
      requiredBy = [ "hive-gateway-self-signed-cert.service" ];
      # The store's container, where it runs here. On a hive that reads a
      # store hosted elsewhere no such unit exists and systemd ignores
      # the name, which is the correct behaviour rather than a gap: what
      # this unit actually needs is the store reachable, and on a remote
      # store nothing local can order against that.
      after = [
        "container@${baoCfg.machine}.service"
        "swarm-bao-services-issuer-policy.service"
      ];
      wants = [ "container@${baoCfg.machine}.service" ];
      path = [
        baoDeploy.package
        pkgs.jq
        pkgs.openssl
        pkgs.coreutils
        pkgs.systemd
      ];
      # Sized like the store's own granting units, and for the same
      # reason: under `seal = "shamir"` an operator unseals BY HAND, and
      # the login below fails for as long as that takes. 2880 × 30s is
      # 24h inside a 25h window — `StartLimit*` are `[Unit]` settings, so
      # the window must exceed `RestartSec × burst` or it closes between
      # attempts and the burst is never reached.
      startLimitBurst = 2880;
      startLimitIntervalSec = 90000;
      serviceConfig = {
        Type = "oneshot";
        RemainAfterExit = true;
        UMask = "0077";
        SyslogIdentifier = "swarm-services-cert";
        Restart = "on-failure";
        RestartSec = 30;
      };
      environment = {
        BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
      }
      // lib.optionalAttrs (cfg.baoClientCertFile != null) {
        BAO_CLIENT_CERT = cfg.baoClientCertFile;
      }
      // lib.optionalAttrs (cfg.baoClientKeyFile != null) {
        BAO_CLIENT_KEY = cfg.baoClientKeyFile;
      }
      # Absent means the system trust store, which is what a deployment
      # with a real CA wants and what a self-signed one must not be left
      # with.
      // lib.optionalAttrs (baoDeploy.serverCaFile != null) {
        BAO_CACERT = baoDeploy.serverCaFile;
      };
      script = ''
        set -euo pipefail
        d=${lib.escapeShellArg cfg.stateDir}
        install -d -m 0755 "$d"

        ${leafCoverage}

        if [ "$want_svc" = 0 ]; then
          echo "no swarm service domains configured — nothing to issue"
          exit 0
        fi

        # Same rule as the hive leaf's, at a different cadence: re-issue
        # when the file is missing, within 30 days of expiry, or no
        # longer carrying every configured name. The name check is what
        # makes adding a swarm service take effect on the rebuild that
        # added it rather than whenever the certificate happens to lapse.
        reissue=0
        { [ -s "$svcleaf" ] && [ -s "$d/swarm-services-key.pem" ]; } || reissue=1
        openssl x509 -in "$svcleaf" -noout -checkend 2592000 >/dev/null 2>&1 || reissue=1
        covers "$svcleaf" "$svcNames" || reissue=1
        [ -s "$svcroot" ] || reissue=1

        if [ "$reissue" = 0 ]; then
          echo "swarm-services leaf valid and covering the configured names — leaving it alone"
          exit 0
        fi

        ${
          if cfg.baoClientCertFile == null || cfg.baoClientKeyFile == null then
            ''
              echo "no bao client certificate configured for this host, so the gateway's" >&2
              echo "swarm-services certificate cannot be requested from the store." >&2
              echo "Set services.hyperhive.deploy.hive-controller.tls.baoClient{Cert,Key}File" >&2
              echo "to a leaf the store's CA signed with CN=${baoDeploy.servicesIssuerCommonName}." >&2
              exit 1''
          else
            ""
        }

        # `bao`'s own message is the only thing separating an unreachable
        # store from a refused identity from a role that would not issue
        # these names, and this unit retries on all three — so it reports
        # which one rather than asserting all three in a sentence of ours.
        err="$(mktemp)"
        trap 'rm -f "$err"' EXIT

        # Cert auth is a login, not a transport setting: the
        # `BAO_CLIENT_*` variables only decide which certificate the
        # handshake presents. Without a token `bao` asks its token
        # helper, and that is a `sh` this unit's `path` does not carry —
        # `-token-only` answers on stdout and skips the helper.
        if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
          echo "could not log in to the swarm secret store with this host's services-issuer certificate." >&2
          cat "$err" >&2
          exit 1
        fi
        export BAO_TOKEN

        echo "requesting the swarm-services leaf from the store"
        resp="$(mktemp "$d/swarm-services.json.XXXXXX")"
        trap 'rm -f "$err" "$resp"' EXIT

        # ONE call, and the response carries all three artefacts —
        # certificate, private key and issuing CA. Splitting it into a
        # local CSR plus `pki/sign` would buy a key that never crosses
        # the wire, and cost the thing that makes this unit worth having:
        # the store, not this host, decides what the certificate says.
        if ! bao write -format=json \
            ${lib.escapeShellArg "${baoServicesPkiMount}/issue/${baoServicesPkiRole}"} \
            common_name=${lib.escapeShellArg (builtins.head swarmServiceDomains)} \
            alt_names=${lib.escapeShellArg (lib.concatStringsSep "," swarmServiceDomains)} \
            > "$resp" 2>"$err"; then
          echo "the store refused to issue the swarm-services certificate." >&2
          cat "$err" >&2
          exit 1
        fi

        # Three fields out of ONE issuance, which is why the response is
        # captured and then split rather than asked for three times with
        # `bao write -field=…`: each of those calls would mint a separate
        # certificate, and the key would belong to a different one than
        # the leaf.
        #
        # `jq -r` into a file the shell opened, never into a variable and
        # never into an argv — the private key does not become visible in
        # /proc on its way to disk.
        #
        # `// empty` on every one of them: `jq -r` renders a missing
        # field as the four characters `null`, which is a non-empty file
        # and would sail past the check below into a certificate slot.
        umask 077
        jq -r '.data.private_key // empty' < "$resp" > "$d/swarm-services-key.pem.new"
        jq -r '.data.certificate // empty' < "$resp" > "$d/swarm-services-leaf.new"
        jq -r '.data.issuing_ca  // empty' < "$resp" > "$svcroot.new"

        for f in "$d/swarm-services-key.pem.new" "$d/swarm-services-leaf.new" "$svcroot.new"; do
          if [ ! -s "$f" ]; then
            echo "the store's response was missing a field: $f is empty" >&2
            exit 1
          fi
        done

        # nginx serves this file verbatim. It carries the leaf ALONE and
        # not leaf-plus-issuer, which is the one visible consequence of
        # flattening the chain: the issuer here is a self-signed root, so
        # appending it would ship an anchor a client either already holds
        # — in which case it is redundant — or does not, in which case
        # shipping it proves nothing. The old file appended the services
        # sub-CA because that WAS an intermediate, and a client anchored
        # on the swarm root could not build the middle of the chain by
        # itself.
        mv -f "$d/swarm-services-leaf.new" "$svcleaf"
        mv -f "$d/swarm-services-key.pem.new" "$d/swarm-services-key.pem"
        chmod 0644 "$svcleaf"
        chmod 0600 "$d/swarm-services-key.pem"

        # The anchor, world-readable, beside the leaf it signed. This is
        # the file an operator adds to a browser's trust store, and the
        # one `hive-tls-ca` folds into `trust-bundle.pem`. Public
        # certificate material only — the private half never leaves the
        # store's `pki` mount, which is what `root/generate/internal`
        # buys.
        rootchanged=0
        if ! cmp -s "$svcroot.new" "$svcroot"; then
          rootchanged=1
        fi
        mv -f "$svcroot.new" "$svcroot"
        chmod 0644 "$svcroot"

        # Propagation, for the RETRY path only. Ordered before both of
        # these, so on a normal boot they have not run yet, `is-active`
        # is false, and ordering alone does the work. What this covers is
        # the store coming up hours after the gateway did: nginx serves a
        # *copy* of the leaf, so re-issuing the source changes nothing
        # until the copy is remade.
        #
        # ⚠️ `--no-block`, and it is not a preference. This unit declares
        # `Before=` both of these, so a blocking `systemctl restart`
        # enqueues a job that systemd will not start until this unit is
        # active — and this unit is not active until its ExecStart
        # returns, which is waiting on that job. A deadlock, held until
        # the 24h retry window's `TimeoutStartSec` fires. Queueing the
        # job and letting it run once we exit is the only ordering that
        # terminates.
        #
        # The cost is real and is the reason `hive-tls-resign` does NOT
        # do this: a propagation that fails after we return does not fail
        # us, so the leaf can rotate on disk while nginx keeps the old
        # copy. That unit blocks because nothing orders it before its
        # consumers; this one cannot.
        if systemctl is-active --quiet nginx.service; then
          echo "swarm-services leaf rotated — re-importing and reloading nginx"
          systemctl restart --no-block hive-gateway-self-signed-cert.service
          systemctl reload --no-block nginx.service
        fi

        # The bundle is assembled by `hive-tls-ca`, which is ordered
        # after this unit and therefore already holds the current root on
        # any normal boot. On the retry path it ran a long time ago, so a
        # CHANGED root has to reach it — and only a changed one, or every
        # boot would bounce a unit with nothing to do.
        if [ "$rootchanged" = 1 ] && systemctl is-active --quiet hive-tls-ca.service; then
          echo "the services root changed — rebuilding the trust bundle"
          systemctl restart --no-block hive-tls-ca.service
        fi
      '';
    };

    systemd.timers.hive-tls-resign = {
      description = "Weekly gateway-leaf re-sign and propagation";
      wantedBy = [ "timers.target" ];
      timerConfig = {
        # Run weekly; Persistent=true fires a missed run on next boot if
        # the timer was not active (e.g. the host was off on the scheduled
        # day), preventing a dormant timer from letting the leaf lapse.
        OnCalendar = "weekly";
        Persistent = true;
      };
    };

    # Signal the hive-c0re lifecycle that a hive CA exists: it bind-mounts
    # this file (read-only, public certs ONLY — never a key) into each
    # agent container so agents + their tools can trust the gateway's
    # self-signed leaf, and the meta flake wires the per-agent trust
    # bundle. It is the ANCHOR bundle rather than `ca.pem` for the reason
    # spelled out where the bundle is written above; no key path is ever
    # exposed (an agent that could read one could mint trusted certs).
    systemd.services.hive-c0re.environment.HIVE_TLS_CA_PATH = "${cfg.stateDir}/trust-bundle.pem";

    # The same anchor, named for the swarm-queue clients that need it when
    # they mint a token from authelia over TLS. Declared HERE, beside the
    # bundle, rather than in each consumer's module: the path is this
    # module's fact, and two consumers re-deriving `${stateDir}/…` would be
    # two places to fix the day it moves.
    #
    # ⚠️ This is what was missing. `swarm-queue-client` built a bare
    # `reqwest::Client`, so it trusted only the platform roots and died at
    # `invalid peer certificate: UnknownIssuer` against a swarm whose
    # authelia is signed by the swarm CA — while this very bundle sat on
    # disk, already assembled, already handed to hive-c0re under a different
    # variable name. The anchor was never missing; nothing pointed the queue
    # client at it.
    #
    # Set unconditionally within this module's `active` guard, exactly like
    # the line above: where there is no hive CA this module contributes
    # nothing at all, and the clients then fall back to the platform roots —
    # which is correct for a swarm fronted by a public certificate.
    systemd.services.hive-c0re.environment.HIVE_C0RE_OIDC_CA_FILE = "${cfg.stateDir}/trust-bundle.pem";
    # ⚠️ Gated, where the hive-c0re line above is not, and the asymmetry is
    # the point: hive-c0re runs on every hive, the controller runs on one.
    # Defining an environment key on a unit that does not exist CREATES a
    # unit fragment for it — inert (no `ExecStart`, empty `wantedBy`, never
    # activated) but present on every non-controller hive with a CA. Caught
    # in review on this PR; it evaluates and builds clean either way, which
    # is exactly why it needed a reviewer rather than a check.
    systemd.services.swarm-controller.environment.SWARM_CONTROLLER_OIDC_CA_FILE =
      lib.mkIf hyperhiveCfg.deploy.swarm-controller.enable "${cfg.stateDir}/trust-bundle.pem";
  };
}