{ pkgs, lib, config, ... }: let cfg = config.services.hyperhive.swarm.nats; autheliaCfg = config.services.hyperhive.swarm.authelia; autheliaUrl = autheliaCfg.url; # The account the callout responder authenticates as, and the account # authorized clients are placed in. Two accounts rather than one: an # account is NATS' isolation boundary, so a responder that shares an # account with its clients can be published to by the things it # authorizes. calloutAccount = "AUTH"; clientAccount = "APP"; machine = "swarm-nats"; # Where the responder's credentials live *inside* the container, and the # host path that resolves to. Two names for one location, because the # host is the only place both filesystems are addressable. secretDirInContainer = "/var/lib/swarm-nats-auth"; inContainer = name: "${secretDirInContainer}/${name}"; secretDir = "/var/lib/nixos-containers/${machine}${secretDirInContainer}"; hostPath = name: "${secretDir}/${name}"; # The responder needs all three credentials. Gating on them rather than # on `cfg.enable` keeps a half-configured hive at "queue up, denying # everyone" instead of "unit crash-looping on a missing file". responderConfigured = cfg.calloutUserSeedFile != "" && cfg.calloutIssuerSeedFile != ""; clientSecretSource = "${autheliaCfg.hostClientSecretDir}/${cfg.clientId}.secret"; introspectionUrl = "${toString autheliaUrl}/api/oidc/introspection"; in { # The swarm's message queue: one NATS server, reached by every hive. # # ⚠️ There is deliberately NO gateway vhost here, and this is the first # swarm service where that is true — the next reader will go looking for # one. NATS speaks its own TCP protocol rather than HTTP, so nginx # cannot front it the way it fronts the forge, matrix and authelia. # Cross-hive reach is the wireguard mesh; `gateway.localNames` and the # per-service vhost pattern do not apply. options.services.hyperhive.swarm.nats = { enable = lib.mkOption { type = lib.types.bool; default = false; description = '' Run the swarm's message queue in a `swarm-nats` container on this host. A swarm has one queue, so this belongs on the same host as the rest of the shared services. Off by default, and off means *absent*: no container is created and nothing else in the evaluated config changes. ''; }; # ⚠️ Deliberately NO `package` option, unlike this module's siblings. # `services.nats` upstream does not expose one — it resolves # `pkgs.nats-server` itself — so an option here would either be # ignored or need an overlay to mean anything, and an option that # does not control what it names is worse than its absence. Pin the # build with `nixpkgs.overlays` if you need to. port = lib.mkOption { type = lib.types.port; default = 4222; description = '' TCP port the queue listens on. 4222 is upstream's default and sits outside hyperhive's claimed ranges (dashboard 7000, forge 3000, matrix 8008, every agent in 8100..8999 via FNV-1a hash). ''; }; clientId = lib.mkOption { type = lib.types.str; default = "swarm-nats"; description = '' OAuth2 client id the queue's authentication path identifies itself with. Must match the `id` of the corresponding entry in `services.hyperhive.swarm.authelia.oidc.clients` — which this module contributes for you when both run on this host. ''; }; calloutUserPublicKey = lib.mkOption { type = lib.types.str; default = ""; example = "UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCBBQ"; description = '' Public half of the **user** nkey the auth-callout responder authenticates as. `auth_callout.auth_users` exempts this identity from needing callout approval — it is the one that answers auth requests, so it cannot wait for itself. **That exemption is exactly why it needs a credential of its own**: without one the escape hatch is an open door, and on a container sharing the host netns it is an open door reachable from every agent container. An nkey rather than a password for the same reason `calloutIssuerPublicKey` is: only the public half appears here, and nix renders it into the world-readable store harmlessly. The seed reaches the responder and nothing else, so until the responder exists **nobody can authenticate as this user at all** — which is what makes a hive with no responder genuinely closed rather than merely gated. Required when `enable` is set. ''; }; calloutIssuerPublicKey = lib.mkOption { type = lib.types.str; default = ""; example = "ACYR44YO3XZRZBJIYLI5SL6LOPIW37JTD52LNOBHUE34XMH7N5ABMFJH"; description = '' Public half of the account nkey whose signature the server accepts on a user JWT minted by the auth-callout responder. **A public key, and therefore a value rather than a path** — the deliberate exception to the rule that credentials are `*File` options. It is published to every client that connects and its whole job is to be widely known; the matching *seed* is the secret, is never named here, and reaches only the responder. Required when `enable` is set. Without it the server has no issuer to trust and no client can be authorized — which is the fail-closed state described below, but arrived at by accident rather than on purpose, so it fails at eval instead. ''; }; authPackage = lib.mkOption { type = lib.types.package; defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-nats-auth"; description = '' The auth-callout responder package. ⚠️ Named `authPackage`, not `package`, on purpose: this module deliberately has **no** `package` option for the server itself (see the note above — upstream's `services.nats` resolves `pkgs.nats-server` on its own), so a bare `package` here would read as "the NATS package" and mean something else entirely. ''; }; calloutUserSeedFile = lib.mkOption { type = lib.types.str; default = ""; example = "/run/secrets/swarm-nats-callout-user.seed"; description = '' Absolute host path to the **seed** whose public half is `calloutUserPublicKey`. The responder authenticates to the queue with it. A `str` rather than a `path`, and the reason is not style: a `path`-typed literal is hash-copied into the world-readable nix store at eval time, which is the opposite of what a seed wants. Same discipline as `otel.headersCredential`. Until this is set the responder cannot start, and the queue stays in its fail-closed state — which is the correct behaviour, not a gap. ''; }; calloutIssuerSeedFile = lib.mkOption { type = lib.types.str; default = ""; example = "/run/secrets/swarm-nats-issuer.seed"; description = '' Absolute host path to the **account** seed whose public half is `calloutIssuerPublicKey`. The responder signs the user JWTs it issues with it, so possession of this file is the authority to admit anyone to the queue. A `str` for the same store-leak reason as `calloutUserSeedFile`. ''; }; }; config = lib.mkIf cfg.enable { assertions = [ { # Fail at EVAL, not at boot: a queue that comes up unable to # authenticate anyone presents as every client hanging, which is # several layers from "the operator never set the issuer". assertion = cfg.calloutIssuerPublicKey != ""; message = '' services.hyperhive.swarm.nats.enable requires nats.calloutIssuerPublicKey — the public half of the account nkey that signs user JWTs for this queue. It is public and belongs in config; the matching seed is a secret and is delivered to the callout responder instead. See docs/swarm/secrets.md for which is which. ''; } { # Without this the callout-exempt user has no credential, and # NATS accepts `CONNECT {"user":"auth"}` from anyone. An eval # failure is the only place to catch that: the rendered config is # valid, the server starts, and the hole is invisible until # somebody connects. assertion = cfg.calloutUserPublicKey != ""; message = '' services.hyperhive.swarm.nats.enable requires nats.calloutUserPublicKey — the public half of the user nkey the auth-callout responder authenticates as. It is exempt from callout approval by design, which is exactly why it needs its own credential: a `users` entry with a name and no key authenticates anyone who sends that name. ''; } { assertion = autheliaUrl != null; message = '' services.hyperhive.swarm.nats.enable requires services.hyperhive.swarm.authelia.url — the queue authenticates clients by validating tokens that authelia issued. It defaults to this host's own instance only when this host runs authelia. A hive that federates with a swarm sets it explicitly to wherever that provider lives. ''; } ]; # One declaration, two readers. The queue knows which client id it # authenticates under; making the operator restate it in authelia's # client list would be a second source of truth for a string whose # mismatch is an opaque 401 from the token endpoint. # # `client_credentials` rather than an authorization-code flow: a # queue's clients are daemons with nobody to redirect, so a # non-interactive grant is what makes a hive able to authenticate at # all. No redirect URI exists or is wanted. # # ⚠️ `kind` says that, and an empty `redirectUris` does NOT. This # declaration carried the comment above while rendering as an # interactive client, because authelia permits only the grants a # client names and an omitted `grant_types` means authorization-code # alone. Measured against 4.39.20: the token endpoint answered # `unauthorized_client: The OAuth 2.0 Client is not allowed to use # authorization grant 'client_credentials'`. Introspection — which # is all the responder needs today — worked throughout, which is why # nothing was visibly broken while the comment was untrue. services.hyperhive.swarm.authelia.oidc.clients = lib.mkIf autheliaCfg.enable [ { id = cfg.clientId; description = "HyperHive swarm queue"; kind = "machine"; } ]; containers.swarm-nats = { autoStart = true; ephemeral = false; # Shared host netns, like every sibling container. # # ⚠️ Which is exactly why the server below must refuse everyone # until the callout responder exists: on this netns the queue is # reachable from every agent container on the hive, so an # unauthenticated interim state would be a hole rather than a # rough edge. privateNetwork = false; config = { ... }: { system.stateVersion = "26.05"; # Shared host netns: this container's own firewall.service # would rewrite the HOST ruleset at every boot. The host # firewall owns all filtering. networking.firewall.enable = false; # Keep the host-copied /etc/resolv.conf intact — resolvconf's # host-tracking regenerates it empty, since the host's copy # does not cross the boundary after start. networking.resolvconf.enable = lib.mkForce false; services.nats = { enable = true; # Retention, so a reader can ask "what did this hive last # say?" without anyone keeping a second copy. The upstream # option also wires `settings.jetstream.store_dir = dataDir`; # the container is `ephemeral = false`, so that survives a # restart with no bind mount. # # ⚠️ Losing the store is not a correctness problem here: a # reader then sees nothing for every hive, which is the true # answer until each one publishes again. It degrades to # honesty rather than to a stale "healthy". jetstream = true; serverName = "swarm-nats"; port = cfg.port; settings = { # Two accounts, and the callout user lives in neither of # the accounts it authorizes into. accounts = { # ⚠️ The nkey is not decoration and its absence was a real # hole: a `users` entry carrying only a `user` name has no # credential, and `CONNECT {"user":"auth"}` is then # accepted with no password at all. Since the name is a # literal in this public module, that made the # callout-exempt identity walk-in-able from every # container on the shared netns — the same class of hole # this module exists to close, moved rather than fixed. # Caught in review on the first version of this file. # An nkey and NOTHING else, both halves measured against a # running server rather than reasoned about: # # { user = "auth"; } → `CONNECT {"user":"auth"}` # is accepted with no # credential at all # { user = "auth"; nkey = "U…"; } → refuses to START: # "Nkey users do not take # usernames or passwords" # { nkey = "U…"; } → what this is # # A malformed key is fail-closed too: the server exits with # "Not a valid public nkey for a user" rather than starting # with a hole. So the only way to get a live server here is # a real key whose seed nobody but the responder holds. ${calloutAccount}.users = [ { nkey = cfg.calloutUserPublicKey; } ]; # ⚠️ `services.nats.jetstream = true` gives the SERVER # JetStream; an account gets it only from its own grant. # Measured against a running 2.14.1 with this exact # two-account shape, because the failure is invisible to # any config-rendering check: # # global jetstream only → `nats kv add` from this # account fails `code=503 err_code=10039 jetstream # not enabled for account`, while the server starts # cleanly and logs "Starting JetStream" # + this line → the same command succeeds # # The grant is per-account by design, and that is worth # keeping: the callout account above deliberately does # NOT get it. The responder mints credentials; it has no # business holding stream state. ${clientAccount} = { jetstream = "enabled"; }; }; authorization = { timeout = "2s"; # 🔒 THIS BLOCK IS THE FAIL-CLOSED STATE, and it is the # measured one rather than the obvious one. # # Measured on the pinned nats-server 2.14.1: both # `authorization { }` and `authorization { users: [] }` # accept an anonymous client and answer PONG — they read # like "authorize nobody" and are wide open. An # auth_callout block sets `auth_required` and refuses # every client whose credential no responder has # approved, so a config whose responder does not exist # yet denies everyone. # # `nats-server -t` calls all three valid; it parses, it # does not authenticate. Only running them tells the # difference. # # ⇒ this is both the safe interim state and the final # shape. Nothing here has to be swapped out when the # responder lands beside it — it only starts being able # to say yes. auth_callout = { issuer = cfg.calloutIssuerPublicKey; auth_users = [ cfg.calloutUserPublicKey ]; account = calloutAccount; }; }; }; }; # The auth-callout responder: the half that lets the server # above say *yes*. Without it the `auth_callout` block is a # door nobody can open, which is the deliberate interim state. # # ⚠️ It is gated on the seeds being configured rather than on # `cfg.enable`, so a half-configured hive gets a running, # refusing queue instead of a unit that crash-loops on a # missing file. A queue that denies everyone is a legible # failure; a restart loop is not. systemd.services.swarm-nats-auth = lib.mkIf responderConfigured { description = "swarm queue auth-callout responder"; after = [ "nats.service" ]; requires = [ "nats.service" ]; wantedBy = [ "multi-user.target" ]; serviceConfig = { ExecStart = lib.concatStringsSep " " [ "${cfg.authPackage}/bin/swarm-nats-auth" "--nats-url nats://127.0.0.1:${toString cfg.port}" "--user-seed-file \${CREDENTIALS_DIRECTORY}/callout-user.seed" "--issuer-seed-file \${CREDENTIALS_DIRECTORY}/issuer.seed" "--client-secret-file \${CREDENTIALS_DIRECTORY}/oidc-client.secret" "--client-id ${lib.escapeShellArg cfg.clientId}" # The account admitted clients land in, by NAME: in # server-config mode the server resolves `aud` against its # own `accounts` block, so this and the block above have to # be the same string — which is why both come from one let. "--account ${lib.escapeShellArg clientAccount}" "--introspection-url ${lib.escapeShellArg introspectionUrl}" ]; # Every credential arrives by `LoadCredential` and is named # on the command line only as a **path** — `argv` is # world-readable via /proc//cmdline, so a value there # would be readable by every process on the host netns. LoadCredential = [ "callout-user.seed:${inContainer "callout-user.seed"}" "issuer.seed:${inContainer "issuer.seed"}" "oidc-client.secret:${inContainer "oidc-client.secret"}" ]; DynamicUser = true; Restart = "on-failure"; RestartSec = "5s"; SyslogIdentifier = "swarm-nats-auth"; }; }; # The server binary, so an operator with a shell in here can # run `nats-server -t` against the generated config. The unit # resolves ExecStart through the store path and puts nothing # on PATH. environment.systemPackages = [ pkgs.nats-server ]; }; }; # Deliver the responder's three credentials into the container before # it starts. Same shape as `hive-matrix-oidc-secret`, and for the same # reason it is a copy rather than a `bindMounts` entry: nixos-container # refuses to start when a bind source is missing, so one absent seed # would take down the **whole container including the queue**, not # merely the responder. A far larger blast radius than the fault. systemd.services.swarm-nats-auth-secrets = lib.mkIf responderConfigured { description = "deliver the swarm queue responder's credentials"; before = [ "container@swarm-nats.service" ]; wantedBy = [ "container@swarm-nats.service" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; SyslogIdentifier = "swarm-nats-auth-secrets"; }; path = [ pkgs.coreutils ]; script = '' set -euo pipefail install -d -m 0700 ${lib.escapeShellArg secretDir} install -m 0400 ${lib.escapeShellArg cfg.calloutUserSeedFile} \ ${lib.escapeShellArg (hostPath "callout-user.seed")} install -m 0400 ${lib.escapeShellArg cfg.calloutIssuerSeedFile} \ ${lib.escapeShellArg (hostPath "issuer.seed")} install -m 0400 ${lib.escapeShellArg clientSecretSource} \ ${lib.escapeShellArg (hostPath "oidc-client.secret")} ''; }; }; }