hyperhive/nix/host-modules/swarm-controller.nix

# The swarm-level controller daemon. Per-host opt-in: a swarm has one
# controller, so most hives leave this off and point at the hive that
# runs it. Distinct from hive-c0re, which every hive runs — c0re owns
# the agents on one host, this owns what is true across hives.
#
# Serves HTTP over a unix socket rather than a TCP port: the gateway's
# nginx is the only intended client, it runs on this same host and so
# reaches the socket by path, and a socket that is never bound to an
# address cannot be reached from off-host by mistake.
{
  pkgs,
  lib,
  config,
  ...
}:
let
  cfg = config.services.hyperhive.swarm.controller;
  autheliaCfg = config.services.hyperhive.swarm.authelia;

  # What `swarmctl` needs in order to act on authelia from the host.
  #
  # Only set when authelia actually runs **here**: the controller can be
  # enabled on a host that is not the swarm's SSO host, and in that case
  # the right behaviour is for `swarmctl user add` to fail saying the
  # value is unset. A guessed path would resolve cleanly and write a file
  # nothing reads, which is the failure mode that costs an afternoon.
  autheliaEnv = lib.optionalAttrs autheliaCfg.enable {
    # The CONFIGURED authelia, not whatever is on PATH: the argon2
    # parameters baked into a hash have to match the verifier's.
    SWARMCTL_AUTHELIA_BIN = "${autheliaCfg.package}/bin/authelia";
    SWARMCTL_AUTHELIA_USERS_FILE = autheliaCfg.hostUsersFile;
  };

  natsCfg = config.services.hyperhive.swarm.nats;
  forgeCfg = config.services.hyperhive.swarm.forge;
  uiCfg = config.services.hyperhive.swarm.ui;

  # The controller's own OAuth2 client. It is NOT a hive: the per-hive
  # clients the roster issues belong to hives, and the responder's client
  # belongs to the responder. One identity per principal — the rule is that
  # a principal's credentials all derive from the same identity, not that
  # the swarm has one.
  queueClientId = cfg.queueClientId;

  # `LoadCredential` and not a copy-oneshot, which is where this deliberately
  # differs from the callout responder: that one delivers INTO a container,
  # so it has to copy across a filesystem boundary. The controller is a plain
  # host unit, so systemd can hand it the file directly — fewer moving parts,
  # and the secret never gains a second on-disk copy to forget about.
  #
  # Every value here comes from an option rather than from what happens to
  # run on this host. The queue is not optional for a controller, but
  # *co-location with it* is — "controller needs queue, but it may not run on
  # same host if configured properly by operator" is the rule. The
  # defaults below cover the co-located case so nobody types a path they
  # didn't need to; the assertions cover the split case so nobody gets a
  # daemon that silently never connects.
  queueEnv = {
    SWARM_CONTROLLER_NATS_URL = cfg.queue.natsUrl;
    SWARM_CONTROLLER_OIDC_TOKEN_ENDPOINT = cfg.queue.tokenEndpoint;
    SWARM_CONTROLLER_OIDC_CLIENT_ID = queueClientId;
    # `%d` is systemd's credentials directory: root reads the plaintext at
    # unit start and the daemon's own user sees it 0400, without the unit
    # ever being able to read the rest of whatever directory the secret
    # came from.
    SWARM_CONTROLLER_OIDC_CLIENT_SECRET_FILE = "%d/queue-client.secret";
  };

  # Independent of `queueEnv` on purpose, and now for a simpler reason
  # than when this was written: the queue's own co-location guard
  # (`queueLocal`) is gone — its coordinates are options with assertions,
  # so a split-host controller fails at eval instead of silently losing
  # its identity. Forge access still has nothing to do with where authelia
  # and NATS run, only with whether `cfg.forgeTokenFile` resolves to a real
  # file — which `forgeTokenFile`'s own default already handles
  # (forge-local vs. operator-copied). Gating a second time here would
  # just repeat that option's own logic under a different name.
  #
  # ⚠️ The asymmetry is deliberate: the queue is REQUIRED for a controller
  # and the forge is not, so absence is an error for one and a supported
  # shape for the other.
  forgeEnv = lib.optionalAttrs (cfg.forgeTokenFile != null) {
    SWARM_CONTROLLER_FORGE_URL = "https://${forgeCfg.domain}";
    # Same `%d` shape as the queue secret above — root reads the plaintext
    # at unit start, the daemon's own user sees a 0400 copy.
    SWARM_CONTROLLER_FORGE_TOKEN_FILE = "%d/forge-token";
  };

  # How the forge must address this controller to deliver a swarm-wide
  # webhook. Gated on the swarm UI being served *here*, because that module
  # is what declares the vhost and the `/webhook/forge/` location inside it:
  # absent, nothing outside this host can reach the endpoint.
  #
  # 🔑 The gate is the point, not a detail. The daemon registers hooks only
  # when this is set, and a hook whose `target_url` nothing answers is worse
  # than no hook at all — forgejo keeps the registration, marks every
  # delivery failed, and the hook still reads as configured.
  webhookEnv = lib.optionalAttrs uiCfg.enable {
    SWARM_CONTROLLER_PUBLIC_URL = "https://${uiCfg.domain}";
  };

  # Not a secret to deliver — `swarm-authelia-bridge`'s own bearer check
  # is satisfied by THIS daemon's existing queue OIDC identity
  # (`queueEnv` above): "one identity per principal" already covers this,
  # so there is nothing new to mint or copy, just the bridge's address.
  authBridgeEnv = lib.optionalAttrs (cfg.authBridgeUrl != null) {
    SWARM_CONTROLLER_AUTH_BRIDGE_URL = cfg.authBridgeUrl;
  };

  # Wrapped rather than documented: every one of these values is derived
  # from an option this deployment already set, so making the operator
  # re-supply them on the command line would be asking them to repeat the
  # config back to it — and to get it wrong the day one of them changes.
  swarmctlConfigured = pkgs.symlinkJoin {
    name = "swarmctl-configured";
    paths = [ cfg.swarmctlPackage ];
    nativeBuildInputs = [ pkgs.makeWrapper ];
    postBuild = ''
      wrapProgram $out/bin/swarmctl ${
        lib.concatStringsSep " " (
          lib.mapAttrsToList (name: value: "--set ${name} ${lib.escapeShellArg value}") autheliaEnv
        )
      }
    '';
  };
in
{
  options.services.hyperhive.swarm.controller = {
    queueClientId = lib.mkOption {
      type = lib.types.str;
      readOnly = true;
      default = "swarm-controller";
      description = ''
        The OAuth2 client id the controller presents to the swarm queue.
        Read-only: it is what this module registers, published so the
        auth-callout responder can be told which client may read every
        hive's key without repeating the string.

        The responder decides that from a client id, and a client id it
        does not recognise is **denied**. A denial reaches a NATS client
        as a timeout rather than an error, and a controller that cannot
        read looks exactly like a swarm where no hive has reported yet —
        so a drift between these two spellings is invisible at the point
        it is introduced and misattributed everywhere it shows up.
      '';
    };

    enable = lib.mkOption {
      type = lib.types.bool;
      default = false;
      description = ''
        Run the swarm-controller daemon on this host. Off by default and
        deliberately not derived from `services.hyperhive.enable`: a swarm
        has one controller, so enabling it per hive is a decision about
        swarm topology, not about whether hyperhive is installed.

        `services.hyperhive.enableAllLocalDefaults` does assert it, and
        that is not an exception to the rule above — it is the rule
        applied. That mode says "this box is the whole deployment", which
        answers the topology question outright, where
        `services.hyperhive.enable` alone never can.
      '';
    };

    package = lib.mkOption {
      type = lib.types.package;
      defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-controller";
      description = ''
        swarm-controller package. Wired by default from this flake's own
        package set (see `flake.nix`); override to run a different build.
      '';
    };

    swarmctlPackage = lib.mkOption {
      type = lib.types.package;
      defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarmctl";
      description = ''
        `swarmctl` package — the swarm operator's CLI, installed on this
        host alongside the daemon and wrapped with the paths it needs.

        A separate option from `package` rather than a second binary in
        the same derivation: the CLI runs as root and acts directly,
        the daemon runs unprivileged and serves a socket, and pinning one
        without the other is a thing an operator may legitimately want.
      '';
    };

    socketPath = lib.mkOption {
      type = lib.types.str;
      default = "/run/swarm-controller/controller.sock";
      description = ''
        Unix socket the daemon serves on, and the path the gateway's nginx
        proxies to.

        The **directory** is the access control here, not the socket mode:
        the socket itself is `0666` (nginx runs as another user, and
        `connect(2)` needs write), exactly as hive-c0re publishes the
        per-agent sockets. What keeps that safe is that the directory holds
        exactly one socket and is traverse-only (`0751`) for everyone else.
        Moving this path under a directory that carries anything else —
        `/run/hyperhive`, which holds the host admin socket, above all —
        exposes whatever else lives there to the same reachability.

        Changing this therefore means re-checking what else lives in the
        new directory, not just the daemon.
      '';
    };

    staleAfterSeconds = lib.mkOption {
      type = lib.types.ints.positive;
      default = 120;
      description = ''
        How old a hive's last status snapshot may be before
        `GET /api/hives/status` reports it as `stale` rather than `fresh`.

        This is a statement about how often hives *publish*, not about how
        patient a reader is — set it above the publishing cadence or every
        hive reads stale between offers. It is an option and not a
        constant precisely because that cadence is a property of the
        deployment.

        Freshness is derived when the endpoint is read, never stored, so
        changing this takes effect for the next request; no hive has to
        re-publish anything.
      '';
    };

    queue = {
      natsUrl = lib.mkOption {
        type = lib.types.str;
        default = "";
        example = "nats://queue.example.com:4222";
        description = ''
          Where the controller reaches the swarm queue.

          Empty means unset, which the assertion below refuses — a
          controller with no queue is not a lighter controller.

          `enableAllLocalDefaults` fills this in with loopback, because
          that address is only correct when the queue is on this host:
          its container shares the host netns. That derivation lives with
          the mode rather than here, so this option describes itself
          rather than a deployment shape.
        '';
      };

      tokenEndpoint = lib.mkOption {
        type = lib.types.str;
        default = lib.optionalString (autheliaCfg.url != null) "${autheliaCfg.url}/api/oidc/token";
        defaultText = lib.literalExpression ''"''${swarm.authelia.url}/api/oidc/token"'';
        description = ''
          The OIDC token endpoint the controller mints its own access
          token from.

          Derived from `swarm.authelia.url`, which is the half of the
          authelia module that exists on every host — so this is already
          correct for a remote provider as long as that URL is set.
        '';
      };

      clientSecretFile = lib.mkOption {
        type = lib.types.str;
        default = "";
        example = "/var/lib/secrets/swarm-controller-queue.secret";
        description = ''
          Path on **this** host holding the plaintext of the controller's
          OAuth2 client secret. A path, never a value: the secret would
          otherwise land in the world-readable nix store.

          The controller cannot mint its own — minting happens inside
          authelia's state directory during its first boot — so away from
          that host the operator places the secret and names it here.
          `enableAllLocalDefaults` points this at the minted file, which
          is exactly the case where one exists locally.

          Read by `LoadCredential`, so it needs to be readable by root at
          unit start and nothing more; the daemon's own user never sees
          the original path.
        '';
      };
    };

    links = lib.mkOption {
      type = lib.types.listOf (
        lib.types.submodule {
          options = {
            label = lib.mkOption {
              type = lib.types.str;
              description = "Display label for the link.";
            };
            icon = lib.mkOption {
              type = lib.types.str;
              default = "";
              description = "Optional icon emoji or short glyph.";
            };
            url = lib.mkOption {
              type = lib.types.str;
              description = "Full URL.";
            };
          };
        }
      );
      default = [ ];
      example = lib.literalExpression ''
        [ { label = "Wiki"; icon = "📖"; url = "https://wiki.example.com/"; } ]
      '';
      description = ''
        Quick links to swarm-wide services, surfaced by the swarm UI's
        links menu (`GET /api/links`). Same shape and same
        zero-code-change-to-extend idea as `hyperhive.dashboardLinks`
        (`nix/agent-modules/dashboard-links.nix`), one level up: rather
        than one central hardcoded list, each service's own module
        contributes its own entry when it is actually enabled on this
        host — `swarm-authelia.nix`, `hive-matrix.nix` and
        `hive-forge/default.nix` all do — the same list-merge idiom
        `services.hyperhive.gateway.localNames` already uses. A future
        service module can push its own entry the same way, and an
        operator can add arbitrary extra entries here directly; neither
        needs a swarm-controller or swarm-ui change.

        Only meaningful on the host that actually runs the controller —
        entries contributed on any other host are computed but never
        read. In a swarm that splits `swarm-authelia`/`hive-matrix`/
        `hive-forge` across hosts other than the controller's, this list
        only reflects what is enabled locally; see each contributing
        module's own activation condition.
      '';
    };

    forgeTokenFile = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      # `services.hyperhive.swarm.forge` has no `enable` of its own to
      # check — the module activates on the general
      # `config.services.hyperhive.enable` instead (see
      # `hive-forge/default.nix`'s own `config = lib.mkIf
      # config.services.hyperhive.enable { ... }`), so that's the
      # condition to match here too. Referencing a `forge.enable` that
      # doesn't exist threw `attribute 'enable' missing` on every host
      # that turns swarm-controller on — caught in review, not by
      # `nix flake check` (nothing in its checked combinations forced
      # this particular default to actually evaluate).
      default = if config.services.hyperhive.enable then forgeCfg.hostSwarmControllerTokenFile else null;
      defaultText = lib.literalExpression ''
        forge's own `hostSwarmControllerTokenFile` when this host runs
        hyperhive at all (forge has no separate enable), else null
      '';
      example = "/var/lib/secrets/swarm-controller-forge.token";
      description = ''
        Path to this host's copy of the swarm-controller's forge access
        token (see `hive-forge/default.nix`'s
        `forgejo-swarm-controller-account` + `hive-forge-swarm-controller-token`
        units, which mint and collect it onto forge's own host).

        Defaults to forge's own delivery path on every host running
        hyperhive (forge deploys unconditionally alongside it — see
        `hive-forge/default.nix`, it has no `enable` of its own).
        Override explicitly if forge's actual token file ends up
        somewhere else — copy it out of forge's
        {option}`services.hyperhive.swarm.forge.hostSwarmControllerTokenFile`
        with whatever secret management this deployment already uses,
        the same shape `swarm.nix`'s `clientSecretFile` documents for
        its own cross-host case. `null` means no forge access — the
        daemon logs that and continues without it, the same
        graceful-absence shape the queue coordinates already use.
      '';
    };

    authBridgeUrl = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default = if autheliaCfg.enable then autheliaCfg.bridgeUrl else null;
      defaultText = lib.literalExpression ''
        authelia's own `bridgeUrl` when this host also runs
        `swarm-authelia`, else null
      '';
      example = "http://127.0.0.1:9092";
      description = ''
        Where `swarm-authelia-bridge` (the only writer of authelia's
        users database) answers — see that option's own doc comment for
        the cross-host caveat, since this default is only correct when
        this host also runs `swarm-authelia`.

        No new credential to configure: the bearer token presented to
        the bridge is minted from THIS daemon's own existing queue OIDC
        identity (`queue.*` above) — "one identity per principal"
        already covers it. `null` means no agent-identity support:
        `CreateIdentity` jobs fail with a clear "no auth bridge
        configured here" error rather than the daemon refusing to
        start, the same graceful-absence shape `forgeTokenFile` uses.
      '';
    };
  };

  config = lib.mkIf (config.services.hyperhive.enable && cfg.enable) {
    users.users.swarm-controller = {
      isSystemUser = true;
      group = "swarm-controller";
      description = "hyperhive swarm-controller daemon";
    };
    users.groups.swarm-controller = { };

    # Installed host-wide, not into the daemon's unit: `swarmctl` is run
    # by a human on this box and acts as root, so the daemon's sandbox is
    # exactly what it must not inherit.
    environment.systemPackages = [ swarmctlConfigured ];

    # One declaration, two readers — the controller knows which client id it
    # authenticates under, so making the operator restate it in authelia's
    # client list would be a second source of truth for a string whose
    # mismatch is an opaque 401 from the token endpoint. Same shape as the
    # queue's own client declaration.
    services.hyperhive.swarm.authelia.oidc.clients = lib.mkIf autheliaCfg.enable [
      {
        id = queueClientId;
        description = "HyperHive swarm controller";
        # `client_credentials`: a daemon authenticating as itself, with
        # nobody to redirect. Declared rather than inferred from an empty
        # redirect list, because authelia permits only the grants a client
        # names and an omitted `grant_types` means authorization-code alone.
        kind = "machine";
      }
    ];

    # A controller without a queue is not a lighter controller, it is a
    # broken one: `/api/hives/status` reads the KV, so its whole aggregate
    # half is off. The operator's rule is that the controller needs a queue
    # but may not run on the same host as one, given it is configured
    # properly — so the queue is required and the co-location is not.
    #
    # ⚠️ These fail at EVAL on purpose. The state they replace is worse than
    # an error: with the coordinates absent the daemon started cleanly,
    # served `/api/hives`, and silently never connected — nothing logged,
    # because on an all-local deployment nothing was wrong.
    assertions = [
      {
        assertion = cfg.queue.natsUrl != "";
        message = ''
          services.hyperhive.swarm.controller.queue.natsUrl is unset.

          It defaults to loopback only when this host also runs the queue
          (`services.hyperhive.swarm.nats.enable`). A controller on its
          own host has to be told where the queue is.
        '';
      }
      {
        assertion = cfg.queue.tokenEndpoint != "";
        message = ''
          services.hyperhive.swarm.controller.queue.tokenEndpoint is unset,
          which means services.hyperhive.swarm.authelia.url is null.

          The controller mints its own access token before it may connect
          to the queue, so it needs to know where the swarm's identity
          provider lives — set that URL, or set this endpoint directly.
        '';
      }
      {
        assertion = cfg.queue.clientSecretFile != "";
        message = ''
          services.hyperhive.swarm.controller.queue.clientSecretFile is
          unset.

          It defaults to the file authelia's first-boot generator mints,
          which only exists when authelia runs on this host. Elsewhere the
          operator places the secret and names it here — the controller
          cannot mint its own, because minting happens inside authelia's
          state directory.
        '';
      }
    ];

    systemd.services.swarm-controller = {
      description = "hyperhive swarm-level controller daemon";
      wantedBy = [ "multi-user.target" ];
      after = [ "network.target" ];

      serviceConfig = {
        ExecStart = "${cfg.package}/bin/swarm-controller";

        # The two differ on purpose. The queue credential is
        # unconditional — the assertions above make its path a value that
        # always exists by the time this renders, so there is no "queue is
        # off here" case left for a `mkIf` to express. The forge token
        # stays optional: a controller with no forge access still serves
        # its HTTP surface, and that IS a supported shape.
        #
        # ⚠️ "the path is a value" is not "the file is on disk". The
        # co-located secret is minted by authelia's FIRST BOOT, in another
        # container, and `hostClientSecretDir`'s own description says a
        # consumer has to wait for it. A `LoadCredential=` naming an
        # absolute path that is not there yet is fatal (`243/CREDENTIALS`),
        # so the daemon spent three of systemd's five default starts losing
        # that race on a real boot — two seconds more and it would have hit
        # `start-limit-hit`, which does not self-heal.
        LoadCredential = [
          "queue-client.secret:${cfg.queue.clientSecretFile}"
        ]
        ++ lib.optional (cfg.forgeTokenFile != null) "forge-token:${cfg.forgeTokenFile}";

        # The placeholder default that makes the above non-fatal.
        # `LoadCredential=` takes priority over `SetCredential=`, so this is
        # only ever seen when the file is missing — and in that case systemd
        # starts the unit instead of refusing to. The controller then serves
        # its HTTP surface with the queue unconfigured, which is a supported
        # shape it already knows how to report.
        #
        # ⚠️ THE VALUE MUST BE NON-EMPTY. `SetCredential=<id>:` with an empty
        # value is rejected by systemd's parser — *"Invalid syntax, ignoring"*
        # — so the whole line is dropped and the fail-soft above silently does
        # not exist. Measured with `systemd-analyze verify`: empty is refused,
        # any non-empty value is accepted. This shipped broken and only looked
        # fine because the credential file happened to be present.
        #
        # The word is deliberate rather than arbitrary: it reaches the token
        # request as the client secret, so authelia refuses it and the journal
        # says so in terms an operator can act on.
        #
        # Safe in a unit file precisely because it is not a secret:
        # `SetCredential=` values are readable by unprivileged processes over
        # IPC, so real key material must never appear here.
        SetCredential = [ "queue-client.secret:placeholder-no-secret-file" ];
        User = "swarm-controller";
        Group = "swarm-controller";
        Restart = "on-failure";
        RestartSec = "5s";

        # `/run/swarm-controller` — its own directory, holding only the
        # socket. See `socketPath`'s description for why that is a security
        # property and not tidiness.
        RuntimeDirectory = "swarm-controller";
        # 0751: traverse-only for others, so the gateway's nginx can reach
        # the socket path without being able to list the directory. Same
        # shape (and same reason) as hive-c0re's runtime dir.
        RuntimeDirectoryMode = "0751";
        # Preserved across restarts so the path never vanishes from under a
        # running nginx. The daemon unlinks a stale socket on start, which
        # is what makes preservation safe.
        RuntimeDirectoryPreserve = "yes";

        StateDirectory = "swarm-controller";
        StateDirectoryMode = "0750";

        # Nothing here needs a writable filesystem, real privileges, or a
        # view of the rest of the machine.
        PrivateTmp = true;
        ProtectSystem = "strict";
        ProtectHome = true;
        NoNewPrivileges = true;
        PrivateDevices = true;
        ProtectKernelTunables = true;
        ProtectKernelModules = true;
        ProtectControlGroups = true;

        # `AF_UNIX` for the socket this daemon serves on, plus what its
        # outbound clients need: it mints authelia tokens and calls the forge
        # over HTTPS (`auth.rs`, `forge.rs`) and reaches the queue over NATS
        # (`main.rs`, `status.rs`). `AF_NETLINK` because glibc's
        # `getaddrinfo` opens a netlink socket to enumerate local addresses
        # before it will return one.
        #
        # ⚠️ This list is a CLAIM ABOUT WHAT THE DAEMON DOES, so it goes stale
        # the moment the daemon grows a client — and it goes stale in the
        # worst available way: a blocked family makes `socket()` return
        # EAFNOSUPPORT, i.e. "Address family not supported by protocol", so
        # the error names the protocol and never the sandbox that refused it.
        # This was `AF_UNIX`-only while the daemon merely served its socket;
        # all three clients above arrived later, and the restriction was not
        # revisited. Add the family when you add the client.
        RestrictAddressFamilies = [
          "AF_UNIX"
          "AF_INET"
          "AF_INET6"
          "AF_NETLINK"
        ];
      };

      # Queue coordinates (`queueEnv`), forge coordinates (`forgeEnv`), and
      # the auth-bridge address (`authBridgeEnv`) merge in last. The
      # daemon refuses a PARTIAL set of any of them rather than treating
      # it as absent, which is why each is built as one attrset and never
      # assigned individually.
      #
      # They differ in how absence is prevented: the queue's is checked by
      # the assertions above, because a controller without a queue is
      # broken rather than lighter; forge's and the auth bridge's are
      # genuinely optional and stay gated on their own option resolving.
      environment = {
        SWARM_CONTROLLER_SOCKET = cfg.socketPath;
        # The swarm's hive directory, JSON-encoded — the full directory
        # (this daemon has no "self" hive to exclude, unlike
        # `swarm.peerHives`, `swarm.hives` minus this hive) rather than
        # peers-minus-self. Consumed by `GET /api/hives`
        # (swarm-controller/src/main.rs::load_hives).
        SWARM_CONTROLLER_HIVES = builtins.toJSON (
          lib.mapAttrsToList (name: h: {
            inherit name;
            inherit (h) domain;
          }) config.services.hyperhive.swarm.hives
        );
        # The merged links list — see `links`' description above for who
        # contributes to it. Consumed by `GET /api/links`
        # (swarm-controller/src/main.rs::load_links).
        SWARM_CONTROLLER_LINKS = builtins.toJSON cfg.links;
        # Staleness threshold for `GET /api/hives/status` — see
        # `staleAfterSeconds`' description. Set unconditionally rather
        # than inside `queueEnv`: it is not a queue coordinate, and
        # nothing about it is unsafe to define on a host whose queue is
        # off (the daemon just has nothing to apply it to).
        SWARM_CONTROLLER_STALE_AFTER_SECS = toString cfg.staleAfterSeconds;
      }
      // queueEnv
      // forgeEnv
      // webhookEnv
      // authBridgeEnv;
    };

    # A systemd credential is a SNAPSHOT: it is materialised into `%d` once,
    # at unit start, and never re-read. That is invisible until the file
    # underneath it changes — and two ordinary things change it.
    #
    #   - it ARRIVES LATE. The co-located secret is minted by authelia's
    #     first boot, in another container, which a host unit cannot order
    #     against. Before this, the unit died at `243/CREDENTIALS` and was
    #     rescued only by burning restarts until the file showed up.
    #   - it is ROTATED. `mint_token` deliberately reads the secret file on
    #     every call so a rotation takes effect without a restart — a
    #     snapshot in `%d` quietly defeats that, and nothing reports it.
    #
    # Watching the file closes both: on close-after-write, restart the
    # daemon so it re-snapshots. `PathChanged=` and not `PathExists=`,
    # measured against the semantics rather than guessed — `PathExists=`
    # activates immediately whenever the file is *already there* at unit
    # start, which would restart a perfectly healthy daemon on every boot.
    # `PathChanged=` requires a write, so it cannot do that and cannot spin.
    #
    # ⚠️ Known gap, stated rather than papered over: a secret appearing in
    # the sub-second window between the daemon starting and this unit
    # watching is missed until the next write. Closing it needs
    # `PathExists=`, whose cost is the spurious per-boot restart above.
    systemd.paths.swarm-controller-credential = {
      description = "watch the swarm controller's queue credential";
      wantedBy = [ "multi-user.target" ];
      pathConfig = {
        PathChanged = cfg.queue.clientSecretFile;
        Unit = "swarm-controller-credential.service";
      };
    };

    # `try-restart`, not `restart`: if the daemon is stopped — masked,
    # disabled, or deliberately down — a secret rotation is not a reason to
    # start it. Rotating a credential should never be how a service comes
    # back to life.
    systemd.services.swarm-controller-credential = {
      description = "restart the swarm controller after its queue credential changed";
      serviceConfig = {
        Type = "oneshot";
        ExecStart = "${pkgs.systemd}/bin/systemctl try-restart swarm-controller.service";
      };
    };
  };
}