4365520ada
A hive whose whole swarm is one box had to be handed two nkeys by hand before its queue could authenticate anyone, which is the one deployment shape where nobody else can supply them. `autoGenerateCallout` mints both keypairs on the host on first boot, keeps the seeds at 0600 host-side, and writes only the public halves into a fragment the server reads at start. The all-local mode turns it on; everywhere else the options stay operator-supplied and the fail-closed eval assertions keep their full force. The server config is not rewritten to do this. A wrapper includes upstream's rendered `settings` verbatim plus the runtime fragment, and the fragment wins — measured, along with the property that makes the whole shape safe: the empty strings the options render in auto mode are values `nats-server` refuses to start on, so any field the merge fails to reach fails closed loudly rather than leaving a walk-in-able server. The wrapper, the settings symlink and the fragment are siblings in one runtime directory, and that is forced rather than tidy: NATS resolves an include with filepath.Join against the config file's own directory, which strips a leading slash, so an absolute include silently becomes a relative one and the server never finds it. The includes are therefore bare filenames. That also means nothing in the closure would otherwise name the rendered settings, so the generator's symlink to it is what keeps it from being garbage-collected under a running server. `accounts` and `authorization` are defined once and rendered twice, into `settings` and into the fragment template. Written out separately they would diverge silently and backwards: the fragment is the later definition, so a future edit to `settings` alone would be ignored on exactly the hives that use auto mode. `validateConfig` goes off in auto mode because `nats-server -t` rejects the empty keys at build time; the parse check moves to server start, where the fragment exists. Upstream's own option description names this case.
711 lines
33 KiB
Nix
711 lines
33 KiB
Nix
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
cfg = config.services.hyperhive.swarm.nats;
|
||
autheliaCfg = config.services.hyperhive.swarm.authelia;
|
||
autheliaUrl = autheliaCfg.url;
|
||
|
||
# The account the callout responder authenticates as, and the account
|
||
# authorized clients are placed in. Two accounts rather than one: an
|
||
# account is NATS' isolation boundary, so a responder that shares an
|
||
# account with its clients can be published to by the things it
|
||
# authorizes.
|
||
calloutAccount = "AUTH";
|
||
clientAccount = "APP";
|
||
|
||
machine = "swarm-nats";
|
||
# Where the responder's credentials live *inside* the container, and the
|
||
# host path that resolves to. Two names for one location, because the
|
||
# host is the only place both filesystems are addressable.
|
||
secretDirInContainer = "/var/lib/swarm-nats-auth";
|
||
inContainer = name: "${secretDirInContainer}/${name}";
|
||
secretDir = "/var/lib/nixos-containers/${machine}${secretDirInContainer}";
|
||
hostPath = name: "${secretDir}/${name}";
|
||
|
||
# The responder needs all three credentials. Gating on them rather than
|
||
# on `cfg.enable` keeps a half-configured hive at "queue up, denying
|
||
# everyone" instead of "unit crash-looping on a missing file".
|
||
#
|
||
# In auto mode the seeds are minted on this host before the container
|
||
# starts, so they are configured by construction.
|
||
responderConfigured =
|
||
cfg.autoGenerateCallout || (cfg.calloutUserSeedFile != "" && cfg.calloutIssuerSeedFile != "");
|
||
clientSecretSource = "${autheliaCfg.hostClientSecretDir}/${cfg.clientId}.secret";
|
||
introspectionUrl = "${toString autheliaUrl}/api/oidc/introspection";
|
||
|
||
# Where the responder's seeds actually come from. One name for two
|
||
# origins, so everything downstream stops caring which mode it is in.
|
||
userSeedFile = if cfg.autoGenerateCallout then autoUserSeed else cfg.calloutUserSeedFile;
|
||
issuerSeedFile = if cfg.autoGenerateCallout then autoIssuerSeed else cfg.calloutIssuerSeedFile;
|
||
|
||
# Seeds stay on the host at 0600 and never enter the container or the
|
||
# store: only the responder needs them, and it reads them by
|
||
# `LoadCredential` from here.
|
||
autoSeedDir = "/var/lib/swarm-nats-callout";
|
||
autoUserSeed = "${autoSeedDir}/callout-user.seed";
|
||
autoIssuerSeed = "${autoSeedDir}/issuer.seed";
|
||
|
||
# The runtime config directory. World-readable is correct — everything in
|
||
# it is public (the server publishes these keys to every client that
|
||
# connects) — and it deliberately does not live in the 0700 responder
|
||
# secret dir, which the `nats` user cannot traverse.
|
||
#
|
||
# ⚠️ ALL THREE FILES LIVE IN ONE DIRECTORY, AND THAT IS FORCED, NOT
|
||
# TIDINESS. `include` paths are resolved with Go's `filepath.Join` against
|
||
# the config file's own directory, which **strips a leading slash** — so an
|
||
# absolute include silently becomes a relative one:
|
||
#
|
||
# include "/nix/store/…-nats.conf"
|
||
# → open agents/atlas/…/nix/store/…-nats.conf: no such file
|
||
#
|
||
# Measured, and it is invisible to every eval-level check: the wrapper
|
||
# renders perfectly and the server refuses to start. ⇒ the includes must be
|
||
# bare filenames, which means the wrapper, the settings and the fragment
|
||
# have to be siblings. (Also measured: resolution is relative to the CONFIG
|
||
# FILE, not the process CWD, so an absolute `-c` path is fine.)
|
||
runtimeDir = "/var/lib/nats-callout";
|
||
runtimeWrapper = "${runtimeDir}/nats.conf";
|
||
hostRuntimeDir = "/var/lib/nixos-containers/${machine}${runtimeDir}";
|
||
|
||
natsFormat = pkgs.formats.json { };
|
||
|
||
# Upstream's rendered settings, read back out of the container's own
|
||
# evaluated config and re-rendered with the same generator — so this is the
|
||
# artifact `services.nats` would have used, not a transcription of it.
|
||
# Referencing it here is also what keeps it alive: with `ExecStart` pointed
|
||
# at a runtime path, nothing else in the closure names it, and the string
|
||
# context of this reference is what stops it being garbage-collected out
|
||
# from under a running server.
|
||
#
|
||
# ⚠️ It should be the very same store path as upstream's (same function,
|
||
# same value), but nothing depends on that being true: if it ever diverged,
|
||
# this is still a correct rendering of the same settings.
|
||
renderedSettings = natsFormat.generate "nats.conf" (
|
||
config.containers.${machine}.config.services.nats.settings
|
||
);
|
||
|
||
# The two blocks whose contents depend on the callout keys, defined ONCE
|
||
# and rendered TWICE: into `services.nats.settings` with the operator's
|
||
# values, and into the runtime fragment with placeholders the generator
|
||
# substitutes.
|
||
#
|
||
# 🪤 Do NOT inline these into `settings` and hand-write the fragment. Both
|
||
# files are loaded, and the fragment is the LATER definition, so it WINS
|
||
# (measured — state/probe-3112-override.sh). A hand-written copy would
|
||
# mean a future edit to `settings` alone is silently ignored on exactly
|
||
# the hives that use auto mode: the config looks changed and behaves
|
||
# unchanged. One definition is what makes that impossible rather than
|
||
# merely unlikely.
|
||
calloutBlocks =
|
||
{
|
||
userKey,
|
||
issuerKey,
|
||
}:
|
||
{
|
||
# Two accounts, and the callout user lives in neither of the accounts
|
||
# it authorizes into.
|
||
accounts = {
|
||
# ⚠️ The nkey is not decoration and its absence was a real hole: a
|
||
# `users` entry carrying only a `user` name has no credential, and
|
||
# `CONNECT {"user":"auth"}` is then accepted with no password at
|
||
# all. Since the name is a literal in this public module, that made
|
||
# the callout-exempt identity walk-in-able from every container on
|
||
# the shared netns — the same class of hole this module exists to
|
||
# close, moved rather than fixed. Caught in review on the first
|
||
# version of this file. An nkey and NOTHING else, both halves
|
||
# measured against a running server rather than reasoned about:
|
||
#
|
||
# { user = "auth"; } → `CONNECT {"user":"auth"}`
|
||
# is accepted with no
|
||
# credential at all
|
||
# { user = "auth"; nkey = "U…"; } → refuses to START:
|
||
# "Nkey users do not take
|
||
# usernames or passwords"
|
||
# { nkey = "U…"; } → what this is
|
||
#
|
||
# A malformed key is fail-closed too: the server exits with
|
||
# "Not a valid public nkey for a user" rather than starting with a
|
||
# hole. So the only way to get a live server here is a real key
|
||
# whose seed nobody but the responder holds.
|
||
#
|
||
# 🔒 That last property is what makes auto mode safe. In auto mode
|
||
# this renders as "" — which the server refuses to start on — and
|
||
# the fragment overrides it at runtime. Every field the merge
|
||
# fails to reach therefore keeps a value that FAILS CLOSED LOUDLY,
|
||
# so an incomplete merge can never leave a walk-in-able server.
|
||
${calloutAccount}.users = [ { nkey = userKey; } ];
|
||
# ⚠️ `services.nats.jetstream = true` gives the SERVER JetStream;
|
||
# an account gets it only from its own grant. Measured against a
|
||
# running 2.14.1 with this exact two-account shape, because the
|
||
# failure is invisible to any config-rendering check:
|
||
#
|
||
# global jetstream only → `nats kv add` from this account fails
|
||
# `code=503 err_code=10039 jetstream not enabled for account`,
|
||
# while the server starts cleanly and logs "Starting JetStream"
|
||
# + this line → the same command succeeds
|
||
#
|
||
# The grant is per-account by design, and that is worth keeping:
|
||
# the callout account above deliberately does NOT get it. The
|
||
# responder mints credentials; it has no business holding stream
|
||
# state.
|
||
#
|
||
# ⚠️ This is also why the fragment renders the COMPLETE accounts
|
||
# block rather than only the AUTH entry: if a later definition
|
||
# replaced the block instead of merging into it, a partial fragment
|
||
# would drop this grant, and the result is a healthy-looking server
|
||
# whose every KV op fails at runtime.
|
||
${clientAccount} = {
|
||
jetstream = "enabled";
|
||
};
|
||
};
|
||
|
||
authorization = {
|
||
timeout = "2s";
|
||
# 🔒 THIS BLOCK IS THE FAIL-CLOSED STATE, and it is the measured
|
||
# one rather than the obvious one.
|
||
#
|
||
# Measured on the pinned nats-server 2.14.1: both
|
||
# `authorization { }` and `authorization { users: [] }` accept an
|
||
# anonymous client and answer PONG — they read like "authorize
|
||
# nobody" and are wide open. An auth_callout block sets
|
||
# `auth_required` and refuses every client whose credential no
|
||
# responder has approved, so a config whose responder does not
|
||
# exist yet denies everyone.
|
||
#
|
||
# `nats-server -t` calls all three valid; it parses, it does not
|
||
# authenticate. Only running them tells the difference.
|
||
#
|
||
# ⇒ this is both the safe interim state and the final shape.
|
||
# Nothing here has to be swapped out when the responder lands
|
||
# beside it — it only starts being able to say yes.
|
||
auth_callout = {
|
||
issuer = issuerKey;
|
||
auth_users = [ userKey ];
|
||
account = calloutAccount;
|
||
};
|
||
};
|
||
};
|
||
|
||
# The fragment as nix renders it, with placeholders where the runtime
|
||
# values go. Rendered by the same JSON generator upstream uses, so the
|
||
# fragment is generated rather than transcribed — NATS' config parser
|
||
# accepts JSON, and an `include` of it merges (measured).
|
||
calloutTemplate = natsFormat.generate "swarm-nats-callout-template.conf" (calloutBlocks {
|
||
userKey = "@USER_PUBKEY@";
|
||
issuerKey = "@ISSUER_PUBKEY@";
|
||
});
|
||
in
|
||
{
|
||
# The swarm's message queue: one NATS server, reached by every hive.
|
||
#
|
||
# ⚠️ There is deliberately NO gateway vhost here, and this is the first
|
||
# swarm service where that is true — the next reader will go looking for
|
||
# one. NATS speaks its own TCP protocol rather than HTTP, so nginx
|
||
# cannot front it the way it fronts the forge, matrix and authelia.
|
||
# Cross-hive reach is the wireguard mesh; `gateway.localNames` and the
|
||
# per-service vhost pattern do not apply.
|
||
|
||
options.services.hyperhive.swarm.nats = {
|
||
enable = lib.mkOption {
|
||
type = lib.types.bool;
|
||
default = false;
|
||
description = ''
|
||
Run the swarm's message queue in a `swarm-nats` container on this
|
||
host. A swarm has one queue, so this belongs on the same host as
|
||
the rest of the shared services.
|
||
|
||
Off by default, and off means *absent*: no container is created
|
||
and nothing else in the evaluated config changes.
|
||
'';
|
||
};
|
||
|
||
# ⚠️ Deliberately NO `package` option, unlike this module's siblings.
|
||
# `services.nats` upstream does not expose one — it resolves
|
||
# `pkgs.nats-server` itself — so an option here would either be
|
||
# ignored or need an overlay to mean anything, and an option that
|
||
# does not control what it names is worse than its absence. Pin the
|
||
# build with `nixpkgs.overlays` if you need to.
|
||
|
||
port = lib.mkOption {
|
||
type = lib.types.port;
|
||
default = 4222;
|
||
description = ''
|
||
TCP port the queue listens on. 4222 is upstream's default and
|
||
sits outside hyperhive's claimed ranges (dashboard 7000, forge
|
||
3000, matrix 8008, every agent in 8100..8999 via FNV-1a hash).
|
||
'';
|
||
};
|
||
|
||
clientId = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "swarm-nats";
|
||
description = ''
|
||
OAuth2 client id the queue's authentication path identifies
|
||
itself with. Must match the `id` of the corresponding entry in
|
||
`services.hyperhive.swarm.authelia.oidc.clients` — which this
|
||
module contributes for you when both run on this host.
|
||
'';
|
||
};
|
||
|
||
autoGenerateCallout = lib.mkOption {
|
||
type = lib.types.bool;
|
||
default = false;
|
||
example = true;
|
||
description = ''
|
||
Generate the auth-callout nkeys on this host instead of taking
|
||
them from `calloutUserPublicKey` / `calloutIssuerPublicKey`.
|
||
|
||
A first-boot unit mints both keypairs if they are absent, keeps
|
||
the seeds host-side at `0600`, and writes only the public halves
|
||
into a fragment the server reads. Nothing secret is evaluated,
|
||
so nothing secret reaches the nix store.
|
||
|
||
Off by default: an option says what it is, not what a deployment
|
||
shape implies. The all-local mode turns this on — see
|
||
`services.hyperhive.enableAllLocalDefaults` in
|
||
./local-defaults.nix, which is the one place that decides what
|
||
"the whole swarm is this box" means.
|
||
|
||
Leave it off wherever the queue and its clients are not the same
|
||
operator's problem: the seeds have to be distributed to whoever
|
||
runs the responder, and generating them here would just move
|
||
that distribution somewhere less visible.
|
||
'';
|
||
};
|
||
|
||
calloutUserPublicKey = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
example = "UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCBBQ";
|
||
description = ''
|
||
Public half of the **user** nkey the auth-callout responder
|
||
authenticates as.
|
||
|
||
`auth_callout.auth_users` exempts this identity from needing
|
||
callout approval — it is the one that answers auth requests, so
|
||
it cannot wait for itself. **That exemption is exactly why it
|
||
needs a credential of its own**: without one the escape hatch is
|
||
an open door, and on a container sharing the host netns it is an
|
||
open door reachable from every agent container.
|
||
|
||
An nkey rather than a password for the same reason
|
||
`calloutIssuerPublicKey` is: only the public half appears here,
|
||
and nix renders it into the world-readable store harmlessly. The
|
||
seed reaches the responder and nothing else, so until the
|
||
responder exists **nobody can authenticate as this user at all**
|
||
— which is what makes a hive with no responder genuinely closed
|
||
rather than merely gated.
|
||
|
||
Required when `enable` is set.
|
||
'';
|
||
};
|
||
|
||
calloutIssuerPublicKey = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
example = "ACYR44YO3XZRZBJIYLI5SL6LOPIW37JTD52LNOBHUE34XMH7N5ABMFJH";
|
||
description = ''
|
||
Public half of the account nkey whose signature the server
|
||
accepts on a user JWT minted by the auth-callout responder.
|
||
|
||
**A public key, and therefore a value rather than a path** —
|
||
the deliberate exception to the rule that credentials are
|
||
`*File` options. It is published to every client that connects
|
||
and its whole job is to be widely known; the matching *seed* is
|
||
the secret, is never named here, and reaches only the responder.
|
||
|
||
Required when `enable` is set. Without it the server has no
|
||
issuer to trust and no client can be authorized — which is the
|
||
fail-closed state described below, but arrived at by accident
|
||
rather than on purpose, so it fails at eval instead.
|
||
'';
|
||
};
|
||
|
||
authPackage = lib.mkOption {
|
||
type = lib.types.package;
|
||
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-nats-auth";
|
||
description = ''
|
||
The auth-callout responder package.
|
||
|
||
⚠️ Named `authPackage`, not `package`, on purpose: this module
|
||
deliberately has **no** `package` option for the server itself
|
||
(see the note above — upstream's `services.nats` resolves
|
||
`pkgs.nats-server` on its own), so a bare `package` here would
|
||
read as "the NATS package" and mean something else entirely.
|
||
'';
|
||
};
|
||
|
||
calloutUserSeedFile = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
example = "/run/secrets/swarm-nats-callout-user.seed";
|
||
description = ''
|
||
Absolute host path to the **seed** whose public half is
|
||
`calloutUserPublicKey`. The responder authenticates to the queue
|
||
with it.
|
||
|
||
A `str` rather than a `path`, and the reason is not style: a
|
||
`path`-typed literal is hash-copied into the world-readable nix
|
||
store at eval time, which is the opposite of what a seed wants.
|
||
Same discipline as `otel.headersCredential`.
|
||
|
||
Until this is set the responder cannot start, and the queue
|
||
stays in its fail-closed state — which is the correct behaviour,
|
||
not a gap.
|
||
'';
|
||
};
|
||
|
||
calloutIssuerSeedFile = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
example = "/run/secrets/swarm-nats-issuer.seed";
|
||
description = ''
|
||
Absolute host path to the **account** seed whose public half is
|
||
`calloutIssuerPublicKey`. The responder signs the user JWTs it
|
||
issues with it, so possession of this file is the authority to
|
||
admit anyone to the queue.
|
||
|
||
A `str` for the same store-leak reason as
|
||
`calloutUserSeedFile`.
|
||
'';
|
||
};
|
||
};
|
||
|
||
config = lib.mkIf cfg.enable {
|
||
assertions = [
|
||
{
|
||
# Fail at EVAL, not at boot: a queue that comes up unable to
|
||
# authenticate anyone presents as every client hanging, which is
|
||
# several layers from "the operator never set the issuer".
|
||
assertion = cfg.autoGenerateCallout || cfg.calloutIssuerPublicKey != "";
|
||
message = ''
|
||
services.hyperhive.swarm.nats.enable requires
|
||
nats.calloutIssuerPublicKey — the public half of the account
|
||
nkey that signs user JWTs for this queue.
|
||
|
||
It is public and belongs in config; the matching seed is a
|
||
secret and is delivered to the callout responder instead. See
|
||
docs/swarm/secrets.md for which is which.
|
||
'';
|
||
}
|
||
{
|
||
# Without this the callout-exempt user has no credential, and
|
||
# NATS accepts `CONNECT {"user":"auth"}` from anyone. An eval
|
||
# failure is the only place to catch that: the rendered config is
|
||
# valid, the server starts, and the hole is invisible until
|
||
# somebody connects.
|
||
assertion = cfg.autoGenerateCallout || cfg.calloutUserPublicKey != "";
|
||
message = ''
|
||
services.hyperhive.swarm.nats.enable requires
|
||
nats.calloutUserPublicKey — the public half of the user nkey
|
||
the auth-callout responder authenticates as.
|
||
|
||
It is exempt from callout approval by design, which is exactly
|
||
why it needs its own credential: a `users` entry with a name
|
||
and no key authenticates anyone who sends that name.
|
||
'';
|
||
}
|
||
{
|
||
assertion = autheliaUrl != null;
|
||
message = ''
|
||
services.hyperhive.swarm.nats.enable requires
|
||
services.hyperhive.swarm.authelia.url — the queue authenticates
|
||
clients by validating tokens that authelia issued.
|
||
|
||
It defaults to this host's own instance only when this host
|
||
runs authelia. A hive that federates with a swarm sets it
|
||
explicitly to wherever that provider lives.
|
||
'';
|
||
}
|
||
];
|
||
|
||
# One declaration, two readers. The queue knows which client id it
|
||
# authenticates under; making the operator restate it in authelia's
|
||
# client list would be a second source of truth for a string whose
|
||
# mismatch is an opaque 401 from the token endpoint.
|
||
#
|
||
# `client_credentials` rather than an authorization-code flow: a
|
||
# queue's clients are daemons with nobody to redirect, so a
|
||
# non-interactive grant is what makes a hive able to authenticate at
|
||
# all. No redirect URI exists or is wanted.
|
||
#
|
||
# ⚠️ `kind` says that, and an empty `redirectUris` does NOT. This
|
||
# declaration carried the comment above while rendering as an
|
||
# interactive client, because authelia permits only the grants a
|
||
# client names and an omitted `grant_types` means authorization-code
|
||
# alone. Measured against 4.39.20: the token endpoint answered
|
||
# `unauthorized_client: The OAuth 2.0 Client is not allowed to use
|
||
# authorization grant 'client_credentials'`. Introspection — which
|
||
# is all the responder needs today — worked throughout, which is why
|
||
# nothing was visibly broken while the comment was untrue.
|
||
services.hyperhive.swarm.authelia.oidc.clients = lib.mkIf autheliaCfg.enable [
|
||
{
|
||
id = cfg.clientId;
|
||
description = "HyperHive swarm queue";
|
||
kind = "machine";
|
||
}
|
||
];
|
||
|
||
containers.swarm-nats = {
|
||
autoStart = true;
|
||
ephemeral = false;
|
||
# Shared host netns, like every sibling container.
|
||
#
|
||
# ⚠️ Which is exactly why the server below must refuse everyone
|
||
# until the callout responder exists: on this netns the queue is
|
||
# reachable from every agent container on the hive, so an
|
||
# unauthenticated interim state would be a hole rather than a
|
||
# rough edge.
|
||
privateNetwork = false;
|
||
config =
|
||
{ ... }:
|
||
{
|
||
system.stateVersion = "26.05";
|
||
|
||
# Shared host netns: this container's own firewall.service
|
||
# would rewrite the HOST ruleset at every boot. The host
|
||
# firewall owns all filtering.
|
||
networking.firewall.enable = false;
|
||
# Keep the host-copied /etc/resolv.conf intact — resolvconf's
|
||
# host-tracking regenerates it empty, since the host's copy
|
||
# does not cross the boundary after start.
|
||
networking.resolvconf.enable = lib.mkForce false;
|
||
|
||
services.nats = {
|
||
enable = true;
|
||
|
||
# Retention, so a reader can ask "what did this hive last
|
||
# say?" without anyone keeping a second copy. The upstream
|
||
# option also wires `settings.jetstream.store_dir = dataDir`;
|
||
# the container is `ephemeral = false`, so that survives a
|
||
# restart with no bind mount.
|
||
#
|
||
# ⚠️ Losing the store is not a correctness problem here: a
|
||
# reader then sees nothing for every hive, which is the true
|
||
# answer until each one publishes again. It degrades to
|
||
# honesty rather than to a stale "healthy".
|
||
jetstream = true;
|
||
|
||
serverName = "swarm-nats";
|
||
port = cfg.port;
|
||
# ⚠️ Build-time validation cannot survive auto mode, and this
|
||
# is measured rather than assumed. In auto mode the keys do
|
||
# not exist until the generator runs, so this renders with
|
||
# empty strings, and `nats-server -t` rejects exactly that:
|
||
#
|
||
# User entry requires a user
|
||
# Expected callout user to be a valid public account nkey,
|
||
# got ""
|
||
#
|
||
# ⇒ leaving it on would fail the BUILD of every all-local
|
||
# hive. Upstream's own description of this option names the
|
||
# case: disable it "when the config includes other files".
|
||
#
|
||
# The check is not lost, it MOVES: the server parses both
|
||
# halves at start, with the fragment present, and refuses to
|
||
# come up if either is malformed. Later rather than at build
|
||
# time — say so in the docs; do not paper over it with an
|
||
# assertion of our own, which would be a second system
|
||
# deciding what counts as a valid config.
|
||
validateConfig = !cfg.autoGenerateCallout;
|
||
|
||
# One definition, rendered here with the operator's values and
|
||
# into the runtime fragment with placeholders. See
|
||
# `calloutBlocks` for why it is not written out twice.
|
||
settings = calloutBlocks {
|
||
userKey = cfg.calloutUserPublicKey;
|
||
issuerKey = cfg.calloutIssuerPublicKey;
|
||
};
|
||
};
|
||
|
||
# Auto mode only: point the server at a wrapper that includes
|
||
# upstream's rendered settings *verbatim* plus the runtime
|
||
# fragment. The alternative — teaching this module to render the
|
||
# config itself — would throw away upstream's `settings`
|
||
# rendering, which is where all the reviewed reasoning lives.
|
||
#
|
||
# The wrapper is written by the generator rather than being a
|
||
# store file, because its includes must be siblings of the
|
||
# fragment (see `runtimeDir`). A server started before the
|
||
# generator has run therefore finds no config at all and refuses,
|
||
# which is the same fail-closed direction as every other failure
|
||
# mode here.
|
||
#
|
||
# `mkForce` because `services.nats` defines ExecStart inside an
|
||
# `mkMerge`; a plain override conflicts rather than winning
|
||
# (measured — state/probe-3112-mkforce.sh).
|
||
systemd.services.nats.serviceConfig.ExecStart = lib.mkIf cfg.autoGenerateCallout (
|
||
lib.mkForce "${pkgs.nats-server}/bin/nats-server -c ${runtimeWrapper}"
|
||
);
|
||
|
||
# The auth-callout responder: the half that lets the server
|
||
# above say *yes*. Without it the `auth_callout` block is a
|
||
# door nobody can open, which is the deliberate interim state.
|
||
#
|
||
# ⚠️ It is gated on the seeds being configured rather than on
|
||
# `cfg.enable`, so a half-configured hive gets a running,
|
||
# refusing queue instead of a unit that crash-loops on a
|
||
# missing file. A queue that denies everyone is a legible
|
||
# failure; a restart loop is not.
|
||
systemd.services.swarm-nats-auth = lib.mkIf responderConfigured {
|
||
description = "swarm queue auth-callout responder";
|
||
after = [ "nats.service" ];
|
||
requires = [ "nats.service" ];
|
||
wantedBy = [ "multi-user.target" ];
|
||
serviceConfig = {
|
||
ExecStart = lib.concatStringsSep " " [
|
||
"${cfg.authPackage}/bin/swarm-nats-auth"
|
||
"--nats-url nats://127.0.0.1:${toString cfg.port}"
|
||
"--user-seed-file \${CREDENTIALS_DIRECTORY}/callout-user.seed"
|
||
"--issuer-seed-file \${CREDENTIALS_DIRECTORY}/issuer.seed"
|
||
"--client-secret-file \${CREDENTIALS_DIRECTORY}/oidc-client.secret"
|
||
"--client-id ${lib.escapeShellArg cfg.clientId}"
|
||
# The account admitted clients land in, by NAME: in
|
||
# server-config mode the server resolves `aud` against its
|
||
# own `accounts` block, so this and the block above have to
|
||
# be the same string — which is why both come from one let.
|
||
"--account ${lib.escapeShellArg clientAccount}"
|
||
"--introspection-url ${lib.escapeShellArg introspectionUrl}"
|
||
];
|
||
# Every credential arrives by `LoadCredential` and is named
|
||
# on the command line only as a **path** — `argv` is
|
||
# world-readable via /proc/<pid>/cmdline, so a value there
|
||
# would be readable by every process on the host netns.
|
||
LoadCredential = [
|
||
"callout-user.seed:${inContainer "callout-user.seed"}"
|
||
"issuer.seed:${inContainer "issuer.seed"}"
|
||
"oidc-client.secret:${inContainer "oidc-client.secret"}"
|
||
];
|
||
DynamicUser = true;
|
||
Restart = "on-failure";
|
||
RestartSec = "5s";
|
||
SyslogIdentifier = "swarm-nats-auth";
|
||
};
|
||
};
|
||
|
||
# The server binary, so an operator with a shell in here can
|
||
# run `nats-server -t` against the generated config. The unit
|
||
# resolves ExecStart through the store path and puts nothing
|
||
# on PATH.
|
||
environment.systemPackages = [ pkgs.nats-server ];
|
||
};
|
||
};
|
||
|
||
# Deliver the responder's three credentials into the container before
|
||
# it starts. Same shape as `hive-matrix-oidc-secret`, and for the same
|
||
# reason it is a copy rather than a `bindMounts` entry: nixos-container
|
||
# refuses to start when a bind source is missing, so one absent seed
|
||
# would take down the **whole container including the queue**, not
|
||
# merely the responder. A far larger blast radius than the fault.
|
||
systemd.services.swarm-nats-auth-secrets = lib.mkIf responderConfigured {
|
||
description = "deliver the swarm queue responder's credentials";
|
||
before = [ "container@swarm-nats.service" ];
|
||
wantedBy = [ "container@swarm-nats.service" ];
|
||
# In auto mode the seeds this copies do not exist until the generator
|
||
# has run. `requires` as well as `after`: if minting fails there is
|
||
# nothing to deliver, and a copy that silently succeeds with a stale
|
||
# or absent seed is worse than not running.
|
||
after = lib.optional cfg.autoGenerateCallout "swarm-nats-callout-keys.service";
|
||
requires = lib.optional cfg.autoGenerateCallout "swarm-nats-callout-keys.service";
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
SyslogIdentifier = "swarm-nats-auth-secrets";
|
||
};
|
||
path = [ pkgs.coreutils ];
|
||
script = ''
|
||
set -euo pipefail
|
||
install -d -m 0700 ${lib.escapeShellArg secretDir}
|
||
install -m 0400 ${lib.escapeShellArg userSeedFile} \
|
||
${lib.escapeShellArg (hostPath "callout-user.seed")}
|
||
install -m 0400 ${lib.escapeShellArg issuerSeedFile} \
|
||
${lib.escapeShellArg (hostPath "issuer.seed")}
|
||
install -m 0400 ${lib.escapeShellArg clientSecretSource} \
|
||
${lib.escapeShellArg (hostPath "oidc-client.secret")}
|
||
'';
|
||
};
|
||
|
||
# Auto mode: mint the callout keypairs on this host before anything
|
||
# that needs them exists.
|
||
#
|
||
# ⚠️ ON THE HOST, not inside the container, and that is the decision
|
||
# the container boundary forces. The responder is a separate unit that
|
||
# needs the user SEED; generating inside the container would trap the
|
||
# seed there and require a way back out — a new secret-export path,
|
||
# which is precisely the mechanism this issue exists to avoid
|
||
# inventing. Here, the seeds never leave a 0700 host directory and only
|
||
# the public halves cross the boundary.
|
||
systemd.services.swarm-nats-callout-keys = lib.mkIf cfg.autoGenerateCallout {
|
||
description = "mint the swarm queue's auth-callout nkeys";
|
||
before = [ "container@swarm-nats.service" ];
|
||
wantedBy = [ "container@swarm-nats.service" ];
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
SyslogIdentifier = "swarm-nats-callout-keys";
|
||
};
|
||
path = [
|
||
pkgs.coreutils
|
||
pkgs.nkeys
|
||
];
|
||
script = ''
|
||
set -euo pipefail
|
||
umask 077
|
||
install -d -m 0700 ${lib.escapeShellArg autoSeedDir}
|
||
|
||
# Mint iff absent. Idempotence is the whole contract: this runs on
|
||
# every boot, and regenerating would silently invalidate every
|
||
# credential the responder has already issued against the old
|
||
# issuer.
|
||
if [ ! -s ${lib.escapeShellArg autoUserSeed} ]; then
|
||
nk -gen user > ${lib.escapeShellArg autoUserSeed}.tmp
|
||
mv ${lib.escapeShellArg autoUserSeed}.tmp ${lib.escapeShellArg autoUserSeed}
|
||
fi
|
||
if [ ! -s ${lib.escapeShellArg autoIssuerSeed} ]; then
|
||
nk -gen account > ${lib.escapeShellArg autoIssuerSeed}.tmp
|
||
mv ${lib.escapeShellArg autoIssuerSeed}.tmp ${lib.escapeShellArg autoIssuerSeed}
|
||
fi
|
||
chmod 0600 ${lib.escapeShellArg autoUserSeed} ${lib.escapeShellArg autoIssuerSeed}
|
||
|
||
user_pub="$(nk -inkey ${lib.escapeShellArg autoUserSeed} -pubout)"
|
||
issuer_pub="$(nk -inkey ${lib.escapeShellArg autoIssuerSeed} -pubout)"
|
||
|
||
# Public halves only — world-readable on purpose, since the server
|
||
# publishes them to every client that connects.
|
||
install -d -m 0755 ${lib.escapeShellArg hostRuntimeDir}
|
||
sed -e "s|@USER_PUBKEY@|$user_pub|g" \
|
||
-e "s|@ISSUER_PUBKEY@|$issuer_pub|g" \
|
||
${calloutTemplate} > ${lib.escapeShellArg hostRuntimeDir}/callout.conf.tmp
|
||
# Mode BEFORE the rename: a rename publishes whatever the file
|
||
# already is, so setting it afterwards leaves a window where the
|
||
# live path has the wrong mode. Same trap as the gateway's
|
||
# atomic-publish path.
|
||
chmod 0444 ${lib.escapeShellArg hostRuntimeDir}/callout.conf.tmp
|
||
mv ${lib.escapeShellArg hostRuntimeDir}/callout.conf.tmp \
|
||
${lib.escapeShellArg hostRuntimeDir}/callout.conf
|
||
|
||
# Upstream's rendered settings, as a sibling the wrapper can name
|
||
# without a leading slash. Refreshed unconditionally — unlike the
|
||
# seeds, this one MUST track the current system, and a stale copy
|
||
# would silently run yesterday's config.
|
||
ln -sfn ${renderedSettings} ${lib.escapeShellArg hostRuntimeDir}/settings.conf
|
||
|
||
# The wrapper. Bare filenames on purpose: NATS resolves an include
|
||
# against the config file's own directory and strips a leading
|
||
# slash, so an absolute path here would silently fail to open.
|
||
# printf, not a heredoc: this lives inside a nix indented string, so
|
||
# a heredoc's terminator depends on nix's indentation stripping and
|
||
# would break the moment `nix fmt` re-indented this block.
|
||
printf 'include "settings.conf"\ninclude "callout.conf"\n' \
|
||
> ${lib.escapeShellArg hostRuntimeDir}/nats.conf.tmp
|
||
chmod 0444 ${lib.escapeShellArg hostRuntimeDir}/nats.conf.tmp
|
||
mv ${lib.escapeShellArg hostRuntimeDir}/nats.conf.tmp \
|
||
${lib.escapeShellArg hostRuntimeDir}/nats.conf
|
||
'';
|
||
};
|
||
};
|
||
}
|