fix(#3343): make the queue a swarm-required service

enableRequiredServices asserts the services that exist once per swarm and
are optional -- its own description says so -- and the queue meets that
rule. It was left out because this file predates the swarm-nats container
by nine days and was never revisited, not because anyone decided against
it.

The all-local mode already derives nats.autoGenerateCallout, so it was
minting the queue's callout nkeys and never starting the queue. With this
the mode's loopback derivation for the controller points at something
that is actually running.

nix/host-modules/local-defaults.nix

@@ -88,5 +88,33 @@ in
     # instead would be wrong: a hive in a larger swarm can legitimately
     # want the shared services without being the host that controls them.
     controller.enable = lib.mkDefault cfg.enableAllLocalDefaults;
+
+    # The controller's queue coordinates. Co-location is what makes these
+    # derivable at all — loopback only reaches the queue when the queue is
+    # here, and the minted client secret only exists on the host authelia
+    # ran its first boot on — so they belong to the mode that asserts this
+    # box is the whole deployment, not to the options' own `default`.
+    #
+    # Deriving them from `swarm.nats.enable` / `swarm.authelia.enable`
+    # inside those defaults is the mixing this file exists to prevent: the
+    # option would be describing a deployment shape instead of describing
+    # itself, and "what does all-local turn on?" would stop having one
+    # answer.
+    #
+    # ⚠️ Must live INSIDE this attrset, not as a second
+    # `config.services.hyperhive.swarm.…` path beside it — written that
+    # way the two definitions of `swarm` collide and the nested one is
+    # silently lost. The gate caught exactly that: mode on, `natsUrl`
+    # still "".
+    #
+    # The *requirement* stays in `swarm-controller.nix` as an assertion:
+    # needing a queue is the controller's own property in every topology,
+    # and only the convenience is local.
+    controller.queue.natsUrl = lib.mkIf cfg.enableAllLocalDefaults (
+      lib.mkDefault "nats://127.0.0.1:${toString config.services.hyperhive.swarm.nats.port}"
+    );
+    controller.queue.clientSecretFile = lib.mkIf cfg.enableAllLocalDefaults (
+      lib.mkDefault "${config.services.hyperhive.swarm.authelia.hostClientSecretDir}/swarm-controller.secret"
+    );
   };
 }

nix/host-modules/swarm-controller.nix

@@ -43,39 +43,43 @@ let
   # the swarm has one.
   queueClientId = "swarm-controller";
 
-  # Both halves have to be here: authelia to have minted the secret, and the
-  # queue to connect to. Same guard, and the same reasoning, as `autheliaEnv`
-  # above — a value set on a host that runs neither would point at a file
-  # that does not exist and produce a daemon that retries forever.
-  queueLocal = autheliaCfg.enable && natsCfg.enable;
-
   # `LoadCredential` and not a copy-oneshot, which is where this deliberately
   # differs from the callout responder: that one delivers INTO a container,
   # so it has to copy across a filesystem boundary. The controller is a plain
   # host unit, so systemd can hand it the file directly — fewer moving parts,
   # and the secret never gains a second on-disk copy to forget about.
-  queueEnv = lib.optionalAttrs queueLocal {
-    # The queue container shares the host netns, so loopback is correct here
-    # and is not the `localhost`-means-the-wrong-thing trap that applies
-    # inside agent containers.
-    SWARM_CONTROLLER_NATS_URL = "nats://127.0.0.1:${toString natsCfg.port}";
-    SWARM_CONTROLLER_OIDC_TOKEN_ENDPOINT = "${autheliaCfg.url}/api/oidc/token";
+  #
+  # Every value here comes from an option rather than from what happens to
+  # run on this host. The queue is not optional for a controller, but
+  # *co-location with it* is — "controller needs queue, but it may not run on
+  # same host if configured properly by operator" is the rule. The
+  # defaults below cover the co-located case so nobody types a path they
+  # didn't need to; the assertions cover the split case so nobody gets a
+  # daemon that silently never connects.
+  queueEnv = {
+    SWARM_CONTROLLER_NATS_URL = cfg.queue.natsUrl;
+    SWARM_CONTROLLER_OIDC_TOKEN_ENDPOINT = cfg.queue.tokenEndpoint;
     SWARM_CONTROLLER_OIDC_CLIENT_ID = queueClientId;
     # `%d` is systemd's credentials directory: root reads the plaintext at
     # unit start and the daemon's own user sees it 0400, without the unit
-    # ever being able to read the rest of authelia's state dir.
+    # ever being able to read the rest of whatever directory the secret
+    # came from.
     SWARM_CONTROLLER_OIDC_CLIENT_SECRET_FILE = "%d/queue-client.secret";
   };
 
-  # Independent of `queueEnv`/`queueLocal` on purpose — the queue
-  # coordinates' own co-location guard is narrower than it looks (it
-  # silently drops the controller's identity on a host that splits
-  # authelia/NATS from swarm-controller; a known gap tracked separately).
-  # Forge access has nothing to do with whether authelia+NATS happen to be
-  # local, only with whether `cfg.forgeTokenFile` resolves to a real file —
-  # which `forgeTokenFile`'s own default already handles (forge-local vs.
-  # operator-copied). Gating a second time here would just repeat that
-  # option's own logic under a different name.
+  # Independent of `queueEnv` on purpose, and now for a simpler reason
+  # than when this was written: the queue's own co-location guard
+  # (`queueLocal`) is gone — its coordinates are options with assertions,
+  # so a split-host controller fails at eval instead of silently losing
+  # its identity. Forge access still has nothing to do with where authelia
+  # and NATS run, only with whether `cfg.forgeTokenFile` resolves to a real
+  # file — which `forgeTokenFile`'s own default already handles
+  # (forge-local vs. operator-copied). Gating a second time here would
+  # just repeat that option's own logic under a different name.
+  #
+  # ⚠️ The asymmetry is deliberate: the queue is REQUIRED for a controller
+  # and the forge is not, so absence is an error for one and a supported
+  # shape for the other.
   forgeEnv = lib.optionalAttrs (cfg.forgeTokenFile != null) {
     SWARM_CONTROLLER_FORGE_URL = "https://${forgeCfg.domain}";
     # Same `%d` shape as the queue secret above — root reads the plaintext
@@ -182,6 +186,61 @@ in
       '';
     };
 
+    queue = {
+      natsUrl = lib.mkOption {
+        type = lib.types.str;
+        default = "";
+        example = "nats://queue.example.com:4222";
+        description = ''
+          Where the controller reaches the swarm queue.
+
+          Empty means unset, which the assertion below refuses — a
+          controller with no queue is not a lighter controller.
+
+          `enableAllLocalDefaults` fills this in with loopback, because
+          that address is only correct when the queue is on this host:
+          its container shares the host netns. That derivation lives with
+          the mode rather than here, so this option describes itself
+          rather than a deployment shape.
+        '';
+      };
+
+      tokenEndpoint = lib.mkOption {
+        type = lib.types.str;
+        default = lib.optionalString (autheliaCfg.url != null) "${autheliaCfg.url}/api/oidc/token";
+        defaultText = lib.literalExpression ''"''${swarm.authelia.url}/api/oidc/token"'';
+        description = ''
+          The OIDC token endpoint the controller mints its own access
+          token from.
+
+          Derived from `swarm.authelia.url`, which is the half of the
+          authelia module that exists on every host — so this is already
+          correct for a remote provider as long as that URL is set.
+        '';
+      };
+
+      clientSecretFile = lib.mkOption {
+        type = lib.types.str;
+        default = "";
+        example = "/var/lib/secrets/swarm-controller-queue.secret";
+        description = ''
+          Path on **this** host holding the plaintext of the controller's
+          OAuth2 client secret. A path, never a value: the secret would
+          otherwise land in the world-readable nix store.
+
+          The controller cannot mint its own — minting happens inside
+          authelia's state directory during its first boot — so away from
+          that host the operator places the secret and names it here.
+          `enableAllLocalDefaults` points this at the minted file, which
+          is exactly the case where one exists locally.
+
+          Read by `LoadCredential`, so it needs to be readable by root at
+          unit start and nothing more; the daemon's own user never sees
+          the original path.
+        '';
+      };
+    };
+
     links = lib.mkOption {
       type = lib.types.listOf (
         lib.types.submodule {
@@ -298,6 +357,53 @@ in
       }
     ];
 
+    # A controller without a queue is not a lighter controller, it is a
+    # broken one: `/api/hives/status` reads the KV, so its whole aggregate
+    # half is off. The operator's rule is that the controller needs a queue
+    # but may not run on the same host as one, given it is configured
+    # properly — so the queue is required and the co-location is not.
+    #
+    # ⚠️ These fail at EVAL on purpose. The state they replace is worse than
+    # an error: with the coordinates absent the daemon started cleanly,
+    # served `/api/hives`, and silently never connected — nothing logged,
+    # because on an all-local deployment nothing was wrong.
+    assertions = [
+      {
+        assertion = cfg.queue.natsUrl != "";
+        message = ''
+          services.hyperhive.swarm.controller.queue.natsUrl is unset.
+
+          It defaults to loopback only when this host also runs the queue
+          (`services.hyperhive.swarm.nats.enable`). A controller on its
+          own host has to be told where the queue is.
+        '';
+      }
+      {
+        assertion = cfg.queue.tokenEndpoint != "";
+        message = ''
+          services.hyperhive.swarm.controller.queue.tokenEndpoint is unset,
+          which means services.hyperhive.swarm.authelia.url is null.
+
+          The controller mints its own access token before it may connect
+          to the queue, so it needs to know where the swarm's identity
+          provider lives — set that URL, or set this endpoint directly.
+        '';
+      }
+      {
+        assertion = cfg.queue.clientSecretFile != "";
+        message = ''
+          services.hyperhive.swarm.controller.queue.clientSecretFile is
+          unset.
+
+          It defaults to the file authelia's first-boot generator mints,
+          which only exists when authelia runs on this host. Elsewhere the
+          operator places the secret and names it here — the controller
+          cannot mint its own, because minting happens inside authelia's
+          state directory.
+        '';
+      }
+    ];
+
     systemd.services.swarm-controller = {
       description = "hyperhive swarm-level controller daemon";
       wantedBy = [ "multi-user.target" ];
@@ -306,14 +412,16 @@ in
       serviceConfig = {
         ExecStart = "${cfg.package}/bin/swarm-controller";
 
-        # Only the credentials that actually resolve on this host. An
-        # absent credential is not a failure for either: the daemon logs
-        # that no queue / no forge is configured and serves its HTTP
-        # surface regardless, which is the correct behaviour on a host
-        # that doesn't run one.
-        LoadCredential =
-          lib.optional queueLocal "queue-client.secret:${autheliaCfg.hostClientSecretDir}/${queueClientId}.secret"
-          ++ lib.optional (cfg.forgeTokenFile != null) "forge-token:${cfg.forgeTokenFile}";
+        # The two differ on purpose. The queue credential is
+        # unconditional — the assertions above make its path a value that
+        # always exists by the time this renders, so there is no "queue is
+        # off here" case left for a `mkIf` to express. The forge token
+        # stays optional: a controller with no forge access still serves
+        # its HTTP surface, and that IS a supported shape.
+        LoadCredential = [
+          "queue-client.secret:${cfg.queue.clientSecretFile}"
+        ]
+        ++ lib.optional (cfg.forgeTokenFile != null) "forge-token:${cfg.forgeTokenFile}";
         User = "swarm-controller";
         Group = "swarm-controller";
         Restart = "on-failure";
@@ -352,10 +460,14 @@ in
       };
 
       # Queue coordinates (`queueEnv`) and forge coordinates (`forgeEnv`)
-      # merge in last, each present only when its own resolution actually
-      # succeeded on this host. Both refuse a PARTIAL set rather than
-      # treating it as absent, which is why each is built as one attrset
-      # and never assigned individually.
+      # merge in last. The daemon refuses a PARTIAL set of either rather
+      # than treating it as absent, which is why each is built as one
+      # attrset and never assigned individually.
+      #
+      # They differ in how absence is prevented: the queue's is checked by
+      # the assertions above, because a controller without a queue is
+      # broken rather than lighter; the forge's is genuinely optional and
+      # stays gated on `forgeTokenFile` resolving.
       environment = {
         SWARM_CONTROLLER_SOCKET = cfg.socketPath;
         # The swarm's hive directory, JSON-encoded — the full directory

nix/host-modules/swarm-required-services.nix

@@ -47,5 +47,16 @@ in
   config.services.hyperhive.swarm = {
     matrix.enable = lib.mkDefault swarmCfg.enableRequiredServices;
     authelia.enable = lib.mkDefault swarmCfg.enableRequiredServices;
+    # The queue. Added later than the two above and missed at the time —
+    # this file predates the `swarm-nats` container by nine days and had
+    # not been revisited since, so its absence was sequence rather than
+    # intent. It meets the rule in the option's own description exactly:
+    # once per swarm, and optional.
+    #
+    # The tell that it was an omission: `local-defaults.nix` already
+    # derives `nats.autoGenerateCallout` from the all-local mode, so that
+    # mode was minting the queue's callout nkeys and then never starting
+    # the queue they authenticate against.
+    nats.enable = lib.mkDefault swarmCfg.enableRequiredServices;
   };
 }