wip: nix unit + secret delivery for the callout responder

flake.nix

@@ -151,6 +151,9 @@
               services.hyperhive.swarm.ui.package =
                 lib.mkDefault
                   self.packages.${pkgs.stdenv.hostPlatform.system}.swarm-ui;
+              services.hyperhive.swarm.nats.authPackage =
+                lib.mkDefault
+                  self.packages.${pkgs.stdenv.hostPlatform.system}.swarm-nats-auth;
               services.hyperhive.gateway.swaggerUiTheme =
                 lib.mkDefault
                   self.packages.${pkgs.stdenv.hostPlatform.system}.swagger-ui-theme;

nix/host-modules/swarm-nats.nix

@@ -16,6 +16,22 @@ let
   # authorizes.
   calloutAccount = "AUTH";
   clientAccount = "APP";
+
+  machine = "swarm-nats";
+  # Where the responder's credentials live *inside* the container, and the
+  # host path that resolves to. Two names for one location, because the
+  # host is the only place both filesystems are addressable.
+  secretDirInContainer = "/var/lib/swarm-nats-auth";
+  inContainer = name: "${secretDirInContainer}/${name}";
+  secretDir = "/var/lib/nixos-containers/${machine}${secretDirInContainer}";
+  hostPath = name: "${secretDir}/${name}";
+
+  # The responder needs all three credentials. Gating on them rather than
+  # on `cfg.enable` keeps a half-configured hive at "queue up, denying
+  # everyone" instead of "unit crash-looping on a missing file".
+  responderConfigured = cfg.calloutUserSeedFile != "" && cfg.calloutIssuerSeedFile != "";
+  clientSecretSource = "${autheliaCfg.hostClientSecretDir}/${cfg.clientId}.secret";
+  introspectionUrl = "${toString autheliaUrl}/api/oidc/introspection";
 in
 {
   # The swarm's message queue: one NATS server, reached by every hive.
@@ -116,6 +132,55 @@ in
         rather than on purpose, so it fails at eval instead.
       '';
     };
+
+    authPackage = lib.mkOption {
+      type = lib.types.package;
+      defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-nats-auth";
+      description = ''
+        The auth-callout responder package.
+
+        ⚠️ Named `authPackage`, not `package`, on purpose: this module
+        deliberately has **no** `package` option for the server itself
+        (see the note above — upstream's `services.nats` resolves
+        `pkgs.nats-server` on its own), so a bare `package` here would
+        read as "the NATS package" and mean something else entirely.
+      '';
+    };
+
+    calloutUserSeedFile = lib.mkOption {
+      type = lib.types.str;
+      default = "";
+      example = "/run/secrets/swarm-nats-callout-user.seed";
+      description = ''
+        Absolute host path to the **seed** whose public half is
+        `calloutUserPublicKey`. The responder authenticates to the queue
+        with it.
+
+        A `str` rather than a `path`, and the reason is not style: a
+        `path`-typed literal is hash-copied into the world-readable nix
+        store at eval time, which is the opposite of what a seed wants.
+        Same discipline as `otel.headersCredential`.
+
+        Until this is set the responder cannot start, and the queue
+        stays in its fail-closed state — which is the correct behaviour,
+        not a gap.
+      '';
+    };
+
+    calloutIssuerSeedFile = lib.mkOption {
+      type = lib.types.str;
+      default = "";
+      example = "/run/secrets/swarm-nats-issuer.seed";
+      description = ''
+        Absolute host path to the **account** seed whose public half is
+        `calloutIssuerPublicKey`. The responder signs the user JWTs it
+        issues with it, so possession of this file is the authority to
+        admit anyone to the queue.
+
+        A `str` for the same store-leak reason as
+        `calloutUserSeedFile`.
+      '';
+    };
   };
 
   config = lib.mkIf cfg.enable {
@@ -276,6 +341,46 @@ in
             };
           };
 
+          # The auth-callout responder: the half that lets the server
+          # above say *yes*. Without it the `auth_callout` block is a
+          # door nobody can open, which is the deliberate interim state.
+          #
+          # ⚠️ It is gated on the seeds being configured rather than on
+          # `cfg.enable`, so a half-configured hive gets a running,
+          # refusing queue instead of a unit that crash-loops on a
+          # missing file. A queue that denies everyone is a legible
+          # failure; a restart loop is not.
+          systemd.services.swarm-nats-auth = lib.mkIf responderConfigured {
+            description = "swarm queue auth-callout responder";
+            after = [ "nats.service" ];
+            requires = [ "nats.service" ];
+            wantedBy = [ "multi-user.target" ];
+            serviceConfig = {
+              ExecStart = lib.concatStringsSep " " [
+                "${cfg.authPackage}/bin/swarm-nats-auth"
+                "--nats-url nats://127.0.0.1:${toString cfg.port}"
+                "--user-seed-file \${CREDENTIALS_DIRECTORY}/callout-user.seed"
+                "--issuer-seed-file \${CREDENTIALS_DIRECTORY}/issuer.seed"
+                "--client-secret-file \${CREDENTIALS_DIRECTORY}/oidc-client.secret"
+                "--client-id ${lib.escapeShellArg cfg.clientId}"
+                "--introspection-url ${lib.escapeShellArg introspectionUrl}"
+              ];
+              # Every credential arrives by `LoadCredential` and is named
+              # on the command line only as a **path** — `argv` is
+              # world-readable via /proc/<pid>/cmdline, so a value there
+              # would be readable by every process on the host netns.
+              LoadCredential = [
+                "callout-user.seed:${inContainer "callout-user.seed"}"
+                "issuer.seed:${inContainer "issuer.seed"}"
+                "oidc-client.secret:${inContainer "oidc-client.secret"}"
+              ];
+              DynamicUser = true;
+              Restart = "on-failure";
+              RestartSec = "5s";
+              SyslogIdentifier = "swarm-nats-auth";
+            };
+          };
+
           # The server binary, so an operator with a shell in here can
           # run `nats-server -t` against the generated config. The unit
           # resolves ExecStart through the store path and puts nothing
@@ -283,5 +388,33 @@ in
           environment.systemPackages = [ pkgs.nats-server ];
         };
     };
+
+    # Deliver the responder's three credentials into the container before
+    # it starts. Same shape as `hive-matrix-oidc-secret`, and for the same
+    # reason it is a copy rather than a `bindMounts` entry: nixos-container
+    # refuses to start when a bind source is missing, so one absent seed
+    # would take down the **whole container including the queue**, not
+    # merely the responder. A far larger blast radius than the fault.
+    systemd.services.swarm-nats-auth-secrets = lib.mkIf responderConfigured {
+      description = "deliver the swarm queue responder's credentials";
+      before = [ "container@swarm-nats.service" ];
+      wantedBy = [ "container@swarm-nats.service" ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+        SyslogIdentifier = "swarm-nats-auth-secrets";
+      };
+      path = [ pkgs.coreutils ];
+      script = ''
+        set -euo pipefail
+        install -d -m 0700 ${lib.escapeShellArg secretDir}
+        install -m 0400 ${lib.escapeShellArg cfg.calloutUserSeedFile} \
+          ${lib.escapeShellArg (hostPath "callout-user.seed")}
+        install -m 0400 ${lib.escapeShellArg cfg.calloutIssuerSeedFile} \
+          ${lib.escapeShellArg (hostPath "issuer.seed")}
+        install -m 0400 ${lib.escapeShellArg clientSecretSource} \
+          ${lib.escapeShellArg (hostPath "oidc-client.secret")}
+      '';
+    };
   };
 }

nix/packages/default.nix

@@ -150,6 +150,12 @@ in
   # Uses the same per-bin extractor, just bound on its own.
   swarm-controller = mkBinPackage "swarm-controller" "hyperhive swarm-level controller daemon";
 
+  # The queue's auth-callout responder. Out of `daemonBins` for the same
+  # reason as the two above and one more: it runs *inside* the swarm-nats
+  # container, not on the host, so it belongs in that container's closure
+  # rather than every hive's.
+  swarm-nats-auth = mkBinPackage "swarm-nats-auth" "hyperhive swarm queue auth-callout responder";
+
   # The swarm operator's CLI, out of `daemonBins` for the same reason as
   # the daemon above: it is installed by the swarm-controller module on
   # the one host that runs the controller, and belongs in that hive's