1d261b3fed
The queue listened in plaintext on 4222, reached by bridge IP or loopback,
and nothing in-tree opened it to another hive. It now has a name, serves a
certificate for that name alone, and refuses clients that do not speak TLS.
- `swarm.nats.domain`, default `nats.<swarm.domain>`, a sibling name like
`swarm.bao.domain`. The queue host answers it via `gateway.localNames`;
every other hive resolves it through the operator's DNS, as for bao.
- `pki/roles/swarm-nats` allows that one name (bare domain, no subdomains,
IPs or localhost, server flag). A `swarm-nats` cert-auth role and policy
may only `update` `pki/issue/swarm-nats`, written by
`swarm-bao-nats-tls-policy`. The login leaf is minted by glue-bao-tls and
paired by glue-nats-bao-identity. `deploy.bao.natsCommonName` is reserved
as a hive name.
- `swarm-bao-nats-tls` issues the leaf into a directory bound read-only into
the container, restarts nats when it rotates, and re-runs daily.
It joins glue-bao-readers-policy-order, so it is ordered after its policy
unit (`after` and `wants`, never `requires`) where the store is on the
same host. The policy unit joins the store's journald list.
- nats gets `tls {}`, with the key via `LoadCredential`, and no
`allow_non_tls`. `validateConfig` is now off in every mode, because the
build-time check loads a leaf that only exists at runtime.
- 4222 is also open on `wg-hive` when the host is on the mesh, never
host-wide.
- `statusPublish.natsUrl`, `queue.agentNatsUrl`, the controller's URL under
`singleHostSwarm`, and the auth responder all dial
`tls://<swarm.nats.domain>:<port>`. swarm-queue-client hands its CA file
to the NATS connection too, so hive-c0re and the controller trust the
leaf's root.
- docs/swarm/README.md: the queue URL and the one DNS record a multi-host
swarm needs.
module-eval-nats-tls pins the role, the policy, the served leaf, the
firewall, the ordering, and a scan of every `*_NATS_URL` and the
responder's URL across the host and its containers.
Closes #4626
1225 lines
58 KiB
Nix
1225 lines
58 KiB
Nix
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
cfg = config.services.hyperhive.swarm.nats;
|
||
autheliaCfg = config.services.hyperhive.swarm.authelia;
|
||
deployCfg = config.services.hyperhive.deploy;
|
||
autheliaUrl = autheliaCfg.url;
|
||
networkCfg = config.services.hyperhive.network;
|
||
# Read even when the controller runs on a different host: what is needed
|
||
# is the client id that module *declares*, which is the same string
|
||
# everywhere, not whether the daemon happens to be enabled here.
|
||
controllerCfg = config.services.hyperhive.swarm.controller;
|
||
|
||
# The account the callout responder authenticates as, and the account
|
||
# authorized clients are placed in. Two accounts rather than one: an
|
||
# account is NATS' isolation boundary, so a responder that shares an
|
||
# account with its clients can be published to by the things it
|
||
# authorizes.
|
||
calloutAccount = "AUTH";
|
||
clientAccount = "APP";
|
||
|
||
machine = "swarm-nats";
|
||
|
||
tlsCfg = config.services.hyperhive.deploy.hive-controller.tls;
|
||
gatewayCfg = config.services.hyperhive.gateway;
|
||
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
||
# The responder introspects authelia over https BY NAME. Its HTTP client is
|
||
# reqwest/rustls, and `rustls-platform-verifier` resolves roots through
|
||
# `rustls-native-certs`, which reads `SSL_CERT_FILE` — so the same assembled
|
||
# bundle the Go containers use applies here. Without it the handshake fails
|
||
# `UnknownIssuer`, introspection fails, and the responder denies *every*
|
||
# client: one missing trust anchor surfacing as `authorization violation` at
|
||
# every would-be queue user.
|
||
caBundleModule = caTrust.trustBundle {
|
||
inherit pkgs;
|
||
name = machine;
|
||
consumers = [ "swarm-nats-auth" ];
|
||
};
|
||
# Where the responder's credentials live *inside* the container, and the
|
||
# host path that resolves to. Two names for one location, because the
|
||
# host is the only place both filesystems are addressable.
|
||
secretDirInContainer = "/var/lib/swarm-nats-auth";
|
||
inContainer = name: "${secretDirInContainer}/${name}";
|
||
secretDir = "/var/lib/nixos-containers/${machine}${secretDirInContainer}";
|
||
hostPath = name: "${secretDir}/${name}";
|
||
|
||
# The responder needs all three credentials. Gating on them rather than
|
||
# on `deployCfg.nats.enable` keeps a half-configured hive at "queue up, denying
|
||
# everyone" instead of "unit crash-looping on a missing file".
|
||
#
|
||
# In auto mode the seeds are minted on this host before the container
|
||
# starts, so they are configured by construction.
|
||
responderConfigured =
|
||
deployCfg.nats.autoGenerateCallout
|
||
|| (deployCfg.nats.calloutUserSeedFile != "" && deployCfg.nats.calloutIssuerSeedFile != "");
|
||
clientSecretSource = "${deployCfg.authelia.hostClientSecretDir}/${cfg.clientId}.secret";
|
||
introspectionUrl = "${toString autheliaUrl}/api/oidc/introspection";
|
||
|
||
# Where the responder's seeds actually come from. One name for two
|
||
# origins, so everything downstream stops caring which mode it is in.
|
||
userSeedFile =
|
||
if deployCfg.nats.autoGenerateCallout then autoUserSeed else deployCfg.nats.calloutUserSeedFile;
|
||
issuerSeedFile =
|
||
if deployCfg.nats.autoGenerateCallout then autoIssuerSeed else deployCfg.nats.calloutIssuerSeedFile;
|
||
|
||
# Seeds stay on the host at 0600 and never enter the container or the
|
||
# store: only the responder needs them, and it reads them by
|
||
# `LoadCredential` from here.
|
||
autoSeedDir = "/var/lib/swarm-nats-callout";
|
||
autoUserSeed = "${autoSeedDir}/callout-user.seed";
|
||
autoIssuerSeed = "${autoSeedDir}/issuer.seed";
|
||
|
||
# The runtime config directory: wrapper, settings symlink and fragment.
|
||
# World-readable is correct (everything in it is public) and it is not in
|
||
# the 0700 responder secret dir, which the `nats` user cannot traverse.
|
||
#
|
||
# ⚠️ ONE DIRECTORY IS FORCED, NOT TIDINESS. NATS resolves an `include` with
|
||
# `filepath.Join(configDir, path)`, which strips a leading slash, so an
|
||
# absolute include silently becomes relative and is never found. The
|
||
# includes must therefore be bare filenames, i.e. siblings. Invisible to
|
||
# eval: the wrapper renders perfectly and the server refuses to start.
|
||
runtimeDir = "/var/lib/nats-callout";
|
||
runtimeWrapper = "${runtimeDir}/nats.conf";
|
||
hostRuntimeDir = "/var/lib/nixos-containers/${machine}${runtimeDir}";
|
||
|
||
natsFormat = pkgs.formats.json { };
|
||
|
||
# Upstream's rendered settings, re-rendered with the same generator on the
|
||
# same value — the artifact `services.nats` would have used, not a
|
||
# transcription. ⚠️ This reference is also the only thing keeping it alive:
|
||
# `ExecStart` names a runtime path, so nothing else in the closure names
|
||
# the store file, and its string context is what stops it being
|
||
# garbage-collected out from under a running server.
|
||
renderedSettings = natsFormat.generate "nats.conf" (
|
||
config.containers.${machine}.config.services.nats.settings
|
||
);
|
||
|
||
# Defined once, rendered twice: into `settings` with the operator's values,
|
||
# and into the runtime fragment with placeholders the generator fills in.
|
||
#
|
||
# 🪤 Do not inline these and hand-write the fragment instead. Both files are
|
||
# loaded and the fragment is the LATER definition, so it wins — a
|
||
# hand-written copy would make a future edit to `settings` silently
|
||
# ineffective on exactly the hives that use auto mode.
|
||
calloutBlocks =
|
||
{
|
||
userKey,
|
||
issuerKey,
|
||
}:
|
||
{
|
||
# Two accounts, and the callout user lives in neither of the accounts
|
||
# it authorizes into.
|
||
accounts = {
|
||
# ⚠️ The nkey is not decoration and its absence was a real hole: a
|
||
# `users` entry carrying only a `user` name has no credential, and
|
||
# `CONNECT {"user":"auth"}` is then accepted with no password at
|
||
# all. Since the name is a literal in this public module, that made
|
||
# the callout-exempt identity walk-in-able from every container on
|
||
# the shared netns — the same class of hole this module exists to
|
||
# close, moved rather than fixed. Caught in review on the first
|
||
# version of this file. An nkey and NOTHING else, both halves
|
||
# measured against a running server rather than reasoned about:
|
||
#
|
||
# { user = "auth"; } → `CONNECT {"user":"auth"}`
|
||
# is accepted with no
|
||
# credential at all
|
||
# { user = "auth"; nkey = "U…"; } → refuses to START:
|
||
# "Nkey users do not take
|
||
# usernames or passwords"
|
||
# { nkey = "U…"; } → what this is
|
||
#
|
||
# A malformed key is fail-closed too: the server exits with
|
||
# "Not a valid public nkey for a user" rather than starting with a
|
||
# hole. So the only way to get a live server here is a real key
|
||
# whose seed nobody but the responder holds.
|
||
#
|
||
# 🔒 That property is what makes auto mode safe: it renders as ""
|
||
# there, so any field the fragment fails to override keeps a value
|
||
# the server refuses to start on. An incomplete merge cannot leave
|
||
# a walk-in-able server.
|
||
${calloutAccount}.users = [ { nkey = userKey; } ];
|
||
# ⚠️ `services.nats.jetstream = true` gives the SERVER JetStream;
|
||
# an account gets it only from its own grant. Measured against a
|
||
# running 2.14.1 with this exact two-account shape, because the
|
||
# failure is invisible to any config-rendering check:
|
||
#
|
||
# global jetstream only → `nats kv add` from this account fails
|
||
# `code=503 err_code=10039 jetstream not enabled for account`,
|
||
# while the server starts cleanly and logs "Starting JetStream"
|
||
# + this line → the same command succeeds
|
||
#
|
||
# The grant is per-account by design, and that is worth keeping:
|
||
# the callout account above deliberately does NOT get it. The
|
||
# responder mints credentials; it has no business holding stream
|
||
# state.
|
||
#
|
||
# ⚠️ Also why the fragment renders the COMPLETE accounts block: if a
|
||
# later definition replaced rather than merged, a partial one would
|
||
# drop this grant and every KV op would fail on a healthy server.
|
||
${clientAccount} = {
|
||
jetstream = "enabled";
|
||
};
|
||
};
|
||
|
||
authorization = {
|
||
timeout = "2s";
|
||
# 🔒 THIS BLOCK IS THE FAIL-CLOSED STATE, and it is the measured
|
||
# one rather than the obvious one.
|
||
#
|
||
# Measured on the pinned nats-server 2.14.1: both
|
||
# `authorization { }` and `authorization { users: [] }` accept an
|
||
# anonymous client and answer PONG — they read like "authorize
|
||
# nobody" and are wide open. An auth_callout block sets
|
||
# `auth_required` and refuses every client whose credential no
|
||
# responder has approved, so a config whose responder does not
|
||
# exist yet denies everyone.
|
||
#
|
||
# `nats-server -t` calls all three valid; it parses, it does not
|
||
# authenticate. Only running them tells the difference.
|
||
#
|
||
# ⇒ this is both the safe interim state and the final shape.
|
||
# Nothing here has to be swapped out when the responder lands
|
||
# beside it — it only starts being able to say yes.
|
||
auth_callout = {
|
||
issuer = issuerKey;
|
||
auth_users = [ userKey ];
|
||
account = calloutAccount;
|
||
};
|
||
};
|
||
};
|
||
|
||
# The fragment as nix renders it, with placeholders where the runtime
|
||
# values go. Rendered by the same JSON generator upstream uses, so the
|
||
# fragment is generated rather than transcribed — NATS' config parser
|
||
# accepts JSON, and an `include` of it merges (measured).
|
||
calloutTemplate = natsFormat.generate "swarm-nats-callout-template.conf" (calloutBlocks {
|
||
userKey = "@USER_PUBKEY@";
|
||
issuerKey = "@ISSUER_PUBKEY@";
|
||
});
|
||
|
||
swarmDomain = config.services.hyperhive.swarm.domain;
|
||
# Total on a null swarm domain, for the reason ./swarm-bao.nix gives.
|
||
domainBase = if swarmDomain == null then "invalid" else swarmDomain;
|
||
|
||
baoCfg = config.services.hyperhive.swarm.bao;
|
||
baoDeploy = deployCfg.bao;
|
||
|
||
# The queue's TLS leaf, bound read-only at the same path on both sides, the
|
||
# shape ./swarm-bao.nix's `tlsDir` uses. The key reaches the server through
|
||
# `LoadCredential`, as openbao's does: it is root-owned 0600 here, and the
|
||
# server runs as `nats`.
|
||
tlsDir = "/var/lib/swarm-nats-tls";
|
||
tlsCertPath = "${tlsDir}/cert.pem";
|
||
tlsKeyPath = "${tlsDir}/key.pem";
|
||
tlsKeyCredential = "tls-key";
|
||
tlsKeyCredentialPath = "/run/credentials/nats.service/${tlsKeyCredential}";
|
||
|
||
# Re-issue once the leaf is past half of the 720h `pki/roles/swarm-nats`
|
||
# grants it (./swarm-bao.nix), so the daily timer below has two weeks of
|
||
# retries before it lapses.
|
||
leafRenewSeconds = 360 * 3600;
|
||
in
|
||
{
|
||
# The swarm's message queue: one NATS server, reached by every hive at
|
||
# `tls://<swarm.nats.domain>:<port>`.
|
||
#
|
||
# ⚠️ There is deliberately NO gateway vhost here. NATS speaks its own TCP
|
||
# protocol rather than HTTP, so nginx cannot front it the way it fronts the
|
||
# forge, matrix and authelia, and the server terminates TLS itself. The
|
||
# name is the half of the sibling pattern that applies, the way it is for
|
||
# bao: `gateway.localNames` on this host, the operator's DNS everywhere
|
||
# else.
|
||
|
||
options.services.hyperhive.swarm.nats = {
|
||
# `enable` moved to `services.hyperhive.deploy.nats.enable` — see
|
||
# ./deploy.nix. What stays here is what the queue IS: its domain,
|
||
# ports, accounts and callout wiring.
|
||
|
||
# ⚠️ Deliberately NO `package` option for the NATS server, anywhere —
|
||
# not here and not under `deploy.nats` either, where every other
|
||
# service's build now lives. `services.nats` upstream does not expose
|
||
# one; it resolves `pkgs.nats-server` itself, so an option would be
|
||
# ignored or need an overlay to mean anything, and an option that does
|
||
# not control what it names is worse than its absence. Pin the build
|
||
# with `nixpkgs.overlays` if you need to. (`deploy.nats.authPackage`
|
||
# is a different thing: the callout responder, which IS ours.)
|
||
|
||
domain = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "nats.${domainBase}";
|
||
defaultText = lib.literalExpression ''"nats.''${services.hyperhive.swarm.domain}"'';
|
||
description = ''
|
||
Name every client reaches the queue on, as
|
||
`tls://<domain>:<port>`, and the only name its certificate carries.
|
||
A **sibling** of the swarm's other service names, for the reason
|
||
{option}`services.hyperhive.swarm.bao.domain` gives.
|
||
|
||
The host running the queue answers it through `gateway.localNames`.
|
||
Every other hive resolves it through the operator's upstream DNS,
|
||
which is where a multi-host swarm needs a record for it.
|
||
|
||
Not in {option}`services.hyperhive.swarm.serviceDomains`: that list
|
||
is the names a gateway vhost fronts, and nginx fronts nothing here.
|
||
'';
|
||
};
|
||
|
||
port = lib.mkOption {
|
||
type = lib.types.port;
|
||
default = 4222;
|
||
description = ''
|
||
TCP port the queue listens on. 4222 is upstream's default and
|
||
sits outside hyperhive's claimed ranges (dashboard 7000, forge
|
||
3000, matrix 8008, every agent in 8100..8999 via FNV-1a hash).
|
||
|
||
Opened on the bridge interface, for agent containers, and on
|
||
`wg-hive` when this host is on the mesh, for other hives. Never
|
||
host-wide. TLS only: a client that does not speak it is refused.
|
||
|
||
Unlike {option}`monitorPort` and {option}`metricsPort`, the
|
||
address is not what bounds who may use this port: the queue's
|
||
`auth_callout` refuses every client it cannot identify.
|
||
'';
|
||
};
|
||
|
||
monitorPort = lib.mkOption {
|
||
type = lib.types.port;
|
||
default = 8222;
|
||
description = ''
|
||
Port NATS serves its **monitoring** endpoint on, bound to
|
||
loopback.
|
||
|
||
Not a metrics endpoint: the server has no Prometheus format of its
|
||
own. This serves `/varz`, `/connz`, `/routez` as JSON, and the
|
||
exporter below is what translates it — which is why enabling the
|
||
exporter without this produces a process that starts cleanly and
|
||
scrapes nothing.
|
||
|
||
⚠️ Loopback, and the exporter is the only intended reader. The
|
||
endpoint is unauthenticated and `/connz` names every connected
|
||
client, so the address it binds is the whole access control. Do
|
||
not widen it, and do not put it behind a gateway vhost expecting
|
||
that to add one.
|
||
'';
|
||
};
|
||
|
||
metricsPort = lib.mkOption {
|
||
type = lib.types.port;
|
||
default = 7777;
|
||
description = ''
|
||
Port the Prometheus exporter serves NATS's metrics on, bound to
|
||
loopback for the swarm collector to scrape.
|
||
|
||
⚠️ This and {option}`monitorPort` are two more claims on a port
|
||
space every swarm container shares — they run in this container but
|
||
`privateNetwork = false`, so a collision with any other hyperhive
|
||
service is a runtime coin toss over which process gets the port,
|
||
with nothing in any log saying so. Both defaults are upstream's
|
||
own (`nats-server` 8222, `prometheus-nats-exporter` 7777) and
|
||
neither is claimed elsewhere in this repo, checked when they were
|
||
added.
|
||
'';
|
||
};
|
||
|
||
clientId = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "swarm-nats";
|
||
description = ''
|
||
OAuth2 client id the queue's authentication path identifies
|
||
itself with. Must match the `id` of the corresponding entry in
|
||
`services.hyperhive.swarm.authelia.oidc.clients` — which this
|
||
module contributes for you when both run on this host.
|
||
'';
|
||
};
|
||
|
||
};
|
||
|
||
# What stays above is what the queue IS to every hive: the ports it answers
|
||
# on and the client id it is registered under. What the host running it
|
||
# decides is below — which responder build it runs, whether it mints its own
|
||
# keypairs, and where the seeds sit. `enable` already lives in ./deploy.nix,
|
||
# which also carries the renames.
|
||
#
|
||
# ⚠️ The two PUBLIC keys move with their seeds rather than staying: peers
|
||
# receive the user key over the wire when they connect (docs/swarm/secrets.md),
|
||
# they never configure it, and splitting a keypair across two namespaces is
|
||
# worse than either placement.
|
||
options.services.hyperhive.deploy.nats = {
|
||
authPackage = lib.mkOption {
|
||
type = lib.types.package;
|
||
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-nats-auth";
|
||
description = ''
|
||
The auth-callout responder package.
|
||
|
||
⚠️ Named `authPackage`, not `package`, on purpose: this module
|
||
deliberately has **no** `package` option for the NATS server
|
||
itself, because upstream's `services.nats` resolves
|
||
`pkgs.nats-server` on its own — see the note above
|
||
`options.services.hyperhive.swarm.nats`. A bare `package` here
|
||
would read as "the NATS package" and mean something else
|
||
entirely.
|
||
'';
|
||
};
|
||
|
||
autoGenerateCallout = lib.mkOption {
|
||
type = lib.types.bool;
|
||
default = false;
|
||
example = true;
|
||
description = ''
|
||
Generate the auth-callout nkeys on this host instead of taking
|
||
them from `calloutUserPublicKey` / `calloutIssuerPublicKey`.
|
||
|
||
A first-boot unit mints both keypairs if absent, keeps the seeds
|
||
host-side at `0600`, and writes only the public halves into a
|
||
fragment the server reads. Nothing secret is evaluated, so
|
||
nothing secret reaches the nix store.
|
||
|
||
Leave it off wherever the queue and its clients are not the same
|
||
operator's problem: the seeds must reach whoever runs the
|
||
responder, and minting them here only moves that distribution
|
||
somewhere less visible. `singleHostSwarm` turns it on.
|
||
'';
|
||
};
|
||
|
||
calloutUserPublicKey = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
example = "UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCBBQ";
|
||
description = ''
|
||
Public half of the **user** nkey the auth-callout responder
|
||
authenticates as.
|
||
|
||
`auth_callout.auth_users` exempts this identity from needing
|
||
callout approval — it is the one that answers auth requests, so
|
||
it cannot wait for itself. **That exemption is exactly why it
|
||
needs a credential of its own**: without one the escape hatch is
|
||
an open door, and it stands open to everything that can reach the
|
||
queue — every process on the host serving it, and every remote
|
||
hive in a multi-host swarm.
|
||
|
||
An nkey rather than a password for the same reason
|
||
`calloutIssuerPublicKey` is: only the public half appears here,
|
||
and nix renders it into the world-readable store harmlessly. The
|
||
seed reaches the responder and nothing else, so until the
|
||
responder exists **nobody can authenticate as this user at all**
|
||
— which is what makes a hive with no responder genuinely closed
|
||
rather than merely gated.
|
||
|
||
Required when `deploy.nats.enable` is set.
|
||
'';
|
||
};
|
||
|
||
calloutIssuerPublicKey = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
example = "ACYR44YO3XZRZBJIYLI5SL6LOPIW37JTD52LNOBHUE34XMH7N5ABMFJH";
|
||
description = ''
|
||
Public half of the account nkey whose signature the server
|
||
accepts on a user JWT minted by the auth-callout responder.
|
||
|
||
**A public key, and therefore a value rather than a path** —
|
||
the deliberate exception to the rule that credentials are
|
||
`*File` options. It is published to every client that connects
|
||
and its whole job is to be widely known; the matching *seed* is
|
||
the secret, is never named here, and reaches only the responder.
|
||
|
||
Required when `deploy.nats.enable` is set. Without it the server
|
||
has no issuer to trust and no client can be authorized — which is
|
||
the fail-closed state described below, but arrived at by accident
|
||
rather than on purpose, so it fails at eval instead.
|
||
'';
|
||
};
|
||
|
||
calloutUserSeedFile = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
example = "/run/secrets/swarm-nats-callout-user.seed";
|
||
description = ''
|
||
Absolute host path to the **seed** whose public half is
|
||
`calloutUserPublicKey`. The responder authenticates to the queue
|
||
with it.
|
||
|
||
A `str` rather than a `path`, and the reason is not style: a
|
||
`path`-typed literal is hash-copied into the world-readable nix
|
||
store at eval time, which is the opposite of what a seed wants.
|
||
Same discipline as `otel.headersCredential`.
|
||
|
||
Until this is set the responder cannot start, and the queue
|
||
stays in its fail-closed state — which is the correct behaviour,
|
||
not a gap.
|
||
'';
|
||
};
|
||
|
||
calloutIssuerSeedFile = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "";
|
||
example = "/run/secrets/swarm-nats-issuer.seed";
|
||
description = ''
|
||
Absolute host path to the **account** seed whose public half is
|
||
`calloutIssuerPublicKey`. The responder signs the user JWTs it
|
||
issues with it, so possession of this file is the authority to
|
||
admit anyone to the queue.
|
||
|
||
A `str` for the same store-leak reason as
|
||
`calloutUserSeedFile`.
|
||
'';
|
||
};
|
||
|
||
baoClientCertFile = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/var/lib/swarm-bao-pki/nats.pem";
|
||
description = ''
|
||
Client certificate `swarm-bao-nats-tls` presents to the swarm's
|
||
secret store when it asks for the queue's TLS leaf. Its subject must
|
||
be {option}`services.hyperhive.deploy.bao.natsCommonName`: cert auth
|
||
matches on the CN, and that role's policy may issue from
|
||
`pki/issue/swarm-nats` and nothing else.
|
||
|
||
No default. ./glue-nats-bao-identity.nix points it at the leaf
|
||
./glue-bao-tls.nix mints, where this host mints one. On a queue host
|
||
without the store, it is a file the operator copies from the store's
|
||
host.
|
||
|
||
A path, never a value.
|
||
'';
|
||
};
|
||
|
||
baoClientKeyFile = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/var/lib/swarm-bao-pki/nats-key.pem";
|
||
description = ''
|
||
Private key for {option}`services.hyperhive.deploy.nats.baoClientCertFile`.
|
||
A path, never a value.
|
||
'';
|
||
};
|
||
};
|
||
|
||
config = lib.mkIf deployCfg.nats.enable {
|
||
# The responder as well as the server: a denial reaches the client as a
|
||
# timeout, so the server's own log is the only place it is an error.
|
||
services.hyperhive.swarm.otel.journaldUnits = [
|
||
"nats"
|
||
"swarm-nats-auth"
|
||
"swarm-bao-nats-tls"
|
||
];
|
||
|
||
# Resolver only: NATS speaks its own protocol, so nginx fronts
|
||
# nothing here — but the auth responder introspects authelia by name.
|
||
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
||
|
||
# The name every client dials, answered with the bridge IP on this host.
|
||
# Other hives resolve it through the operator's DNS, as they do bao's.
|
||
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
||
|
||
assertions = [
|
||
{
|
||
# Fail at EVAL, not at boot: a queue that comes up unable to
|
||
# authenticate anyone presents as every client hanging, which is
|
||
# several layers from "the operator never set the issuer".
|
||
assertion = deployCfg.nats.autoGenerateCallout || deployCfg.nats.calloutIssuerPublicKey != "";
|
||
message = ''
|
||
services.hyperhive.deploy.nats.enable requires
|
||
services.hyperhive.deploy.nats.calloutIssuerPublicKey — the
|
||
public half of the account nkey that signs user JWTs for this
|
||
queue.
|
||
|
||
It is public and belongs in config; the matching seed is a
|
||
secret and is delivered to the callout responder instead. See
|
||
docs/swarm/secrets.md for which is which.
|
||
'';
|
||
}
|
||
{
|
||
# Without this the callout-exempt user has no credential, and
|
||
# NATS accepts `CONNECT {"user":"auth"}` from anyone. An eval
|
||
# failure is the only place to catch that: the rendered config is
|
||
# valid, the server starts, and the hole is invisible until
|
||
# somebody connects.
|
||
assertion = deployCfg.nats.autoGenerateCallout || deployCfg.nats.calloutUserPublicKey != "";
|
||
message = ''
|
||
services.hyperhive.deploy.nats.enable requires
|
||
services.hyperhive.deploy.nats.calloutUserPublicKey — the
|
||
public half of the user nkey the auth-callout responder
|
||
authenticates as.
|
||
|
||
It is exempt from callout approval by design, which is exactly
|
||
why it needs its own credential: a `users` entry with a name
|
||
and no key authenticates anyone who sends that name.
|
||
'';
|
||
}
|
||
{
|
||
# The two above guard the halves the SERVER needs; the responder
|
||
# needs the other halves, and nothing related them. Satisfying
|
||
# only the public ones renders a valid `auth_callout` block and
|
||
# defines no responder — and callout with no responder is the
|
||
# fail-closed state, so the queue refuses everyone.
|
||
#
|
||
# Nothing downstream catches it. `nats-server -t` runs in exactly
|
||
# this case and passes, because the config IS valid; the missing
|
||
# piece is a unit, and the absence of a unit is not an event. The
|
||
# symptom is every client timing out, since a NATS denial reaches
|
||
# the client as a timeout rather than an error.
|
||
# The binding itself, not a copy of its formula: what is being
|
||
# asserted IS "the responder is configured", and two copies of one
|
||
# boolean is two places for a future edit to land in only one.
|
||
assertion = responderConfigured;
|
||
message = ''
|
||
services.hyperhive.deploy.nats has callout public keys but no
|
||
seed files:
|
||
|
||
deploy.nats.calloutUserSeedFile = "${deployCfg.nats.calloutUserSeedFile}"
|
||
deploy.nats.calloutIssuerSeedFile = "${deployCfg.nats.calloutIssuerSeedFile}"
|
||
|
||
Each seed is the private half of the public key already set
|
||
here — the server verifies with the public half, the responder
|
||
signs with the private one. Configuring one side alone leaves
|
||
this queue with an auth-callout nobody answers, which refuses
|
||
every client rather than degrading.
|
||
|
||
Set both seed paths, or set deploy.nats.autoGenerateCallout = true
|
||
to have this host mint all four.
|
||
'';
|
||
}
|
||
];
|
||
|
||
# One declaration, two readers. The queue knows which client id it
|
||
# authenticates under; making the operator restate it in authelia's
|
||
# client list would be a second source of truth for a string whose
|
||
# mismatch is an opaque 401 from the token endpoint.
|
||
#
|
||
# `client_credentials` rather than an authorization-code flow: a
|
||
# queue's clients are daemons with nobody to redirect, so a
|
||
# non-interactive grant is what makes a hive able to authenticate at
|
||
# all. No redirect URI exists or is wanted.
|
||
#
|
||
# ⚠️ `kind` says that, and an empty `redirectUris` does NOT. This
|
||
# declaration carried the comment above while rendering as an
|
||
# interactive client, because authelia permits only the grants a
|
||
# client names and an omitted `grant_types` means authorization-code
|
||
# alone. Measured against 4.39.20: the token endpoint answered
|
||
# `unauthorized_client: The OAuth 2.0 Client is not allowed to use
|
||
# authorization grant 'client_credentials'`. Introspection — which
|
||
# is all the responder needs today — worked throughout, which is why
|
||
# nothing was visibly broken while the comment was untrue.
|
||
services.hyperhive.swarm.authelia.oidc.clients = lib.mkIf deployCfg.authelia.enable [
|
||
{
|
||
id = cfg.clientId;
|
||
description = "HyperHive swarm queue";
|
||
kind = "machine";
|
||
}
|
||
];
|
||
|
||
# Declared here rather than in the collector's module: an entry then
|
||
# exists only where the service that named it runs. Gated on a
|
||
# collector, because a target nobody reads asserts a collection that is
|
||
# not happening.
|
||
#
|
||
# ⚠️ KNOWN LIMITATION, and it is SILENT. That rule constrains the
|
||
# target, not the scraper — nothing places the collector on this host.
|
||
# `swarm-required-services.nix` derives `nats.enable` and `otel.enable`
|
||
# from one `lib.mkDefault`, so they are co-located by *default*, and an
|
||
# operator may split them. Split, NATS is never scraped: this host
|
||
# declares an entry no local collector reads, the collector's host never
|
||
# enabled this module. No error, no warning — a healthy exporter and an
|
||
# empty dashboard.
|
||
#
|
||
# Not guardable: separate hosts are separate evaluations with no shared
|
||
# context, so this one cannot see what that one runs. Saying so is the
|
||
# only mechanism there is.
|
||
#
|
||
# ⚠️ `swarm.otel`, not `hyperhive.otel` — two collectors one word apart,
|
||
# and only this one reads `scrapeTargets`. Written in full so the gate
|
||
# and the option it gates are visibly the same path.
|
||
services.hyperhive.swarm.otel.scrapeTargets =
|
||
lib.mkIf config.services.hyperhive.deploy.swarm-otel.enable
|
||
{
|
||
nats = "127.0.0.1:${toString cfg.metricsPort}";
|
||
};
|
||
|
||
# Open the client port on the bridge, so agent containers can reach
|
||
# the queue. `privateNetwork = false` below means the server binds in
|
||
# the host netns, which is the precondition this option names — but
|
||
# sharing a netns is not reachability: the bridge interface is
|
||
# default-deny, so without this an agent's connect attempt is dropped
|
||
# by the firewall and looks exactly like every other NATS failure,
|
||
# a timeout.
|
||
#
|
||
# ⚠️ Never the world. What makes it safe to open at all is that the
|
||
# queue is fail-closed: `auth_callout` admits nobody until the responder
|
||
# above answers for them, so an agent that reaches this port still has to
|
||
# present a token authelia vouches for.
|
||
#
|
||
# An agent reaches it by name, which dnsmasq answers with the bridge IP.
|
||
services.hyperhive.network.exposeHostPorts = [ cfg.port ];
|
||
|
||
# The other hives' way in, interface-scoped like
|
||
# ./swarm-snapshot-store.nix's receiver. The firewall is default-deny
|
||
# before a packet reaches the listener, so without this nothing in-tree
|
||
# lets another hive reach the queue at all.
|
||
networking.firewall.interfaces.wg-hive.allowedTCPPorts = lib.mkIf deployCfg.wireguard.enable [
|
||
cfg.port
|
||
];
|
||
|
||
# The bind source has to exist before the container starts, leaf or not:
|
||
# nixos-container refuses to start over a missing one, which would take
|
||
# the whole queue down rather than only its TLS.
|
||
systemd.tmpfiles.rules = [ "d ${tlsDir} 0755 root root -" ];
|
||
|
||
containers.swarm-nats = {
|
||
autoStart = true;
|
||
ephemeral = false;
|
||
# Journal files on the host, not inside the container: nixpkgs hardcodes
|
||
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
|
||
extraFlags = [ "--link-journal=host" ];
|
||
# Shared host netns, like every sibling swarm container.
|
||
#
|
||
# ⚠️ Which is exactly why the server below must refuse everyone
|
||
# until the callout responder exists: the queue is on the host's
|
||
# own loopback, in reach of every process there and every sibling
|
||
# on this netns, and each remote hive in a multi-host swarm dials
|
||
# it directly. An unauthenticated interim state would be a hole
|
||
# rather than a rough edge.
|
||
#
|
||
# Not agent containers, though: they have a netns of their own and
|
||
# the bridge firewall does not open this port.
|
||
privateNetwork = false;
|
||
# The public trust bundle (empty when the gateway is not self-signed)
|
||
# and the queue's own TLS leaf, both read-only.
|
||
bindMounts = caTrust.bindMount // {
|
||
${tlsDir} = {
|
||
hostPath = tlsDir;
|
||
isReadOnly = true;
|
||
};
|
||
};
|
||
config =
|
||
{ ... }:
|
||
{
|
||
imports = [
|
||
(import ./swarm-container-resolver.nix {
|
||
inherit (networkCfg) bridgeIp;
|
||
# The responder introspects authelia BY NAME on every auth
|
||
# request, and a responder that cannot resolve it denies
|
||
# every client — so it must not start before the resolver
|
||
# file exists.
|
||
dnsConsumers = [ "swarm-nats-auth.service" ];
|
||
})
|
||
caBundleModule
|
||
];
|
||
|
||
system.stateVersion = "26.05";
|
||
|
||
# Shared host netns: this container's own firewall.service
|
||
# would rewrite the HOST ruleset at every boot. The host
|
||
# firewall owns all filtering.
|
||
networking.firewall.enable = false;
|
||
# resolvconf stays off because the resolver unit imported above
|
||
# owns /etc/resolv.conf. Leaving it on would let host-tracking
|
||
# regenerate the file empty, since the host's copy does not
|
||
# cross the boundary after start.
|
||
networking.resolvconf.enable = lib.mkForce false;
|
||
|
||
services.nats = {
|
||
enable = true;
|
||
|
||
# Retention, so a reader can ask "what did this hive last
|
||
# say?" without anyone keeping a second copy. The upstream
|
||
# option also wires `settings.jetstream.store_dir = dataDir`;
|
||
# the container is `ephemeral = false`, so that survives a
|
||
# restart with no bind mount.
|
||
#
|
||
# ⚠️ Losing the store is not a correctness problem here: a
|
||
# reader then sees nothing for every hive, which is the true
|
||
# answer until each one publishes again. It degrades to
|
||
# honesty rather than to a stale "healthy".
|
||
jetstream = true;
|
||
|
||
serverName = "swarm-nats";
|
||
port = cfg.port;
|
||
# Off in every mode now. The `tls` block below names a leaf that
|
||
# exists only at runtime, and `nats-server -t` loads it, so the
|
||
# build-time check fails on every hive. In auto mode it failed
|
||
# already, on callout keys that are empty until the generator
|
||
# runs. Upstream's own description names the case: disable it
|
||
# when the config includes other files. The check moves to
|
||
# server start.
|
||
validateConfig = false;
|
||
|
||
settings =
|
||
calloutBlocks {
|
||
userKey = deployCfg.nats.calloutUserPublicKey;
|
||
issuerKey = deployCfg.nats.calloutIssuerPublicKey;
|
||
}
|
||
// {
|
||
# The most a single publish may be, in bytes, before the
|
||
# server answers `-ERR 'Maximum Payload Violation'` and
|
||
# closes the connection — not a truncation, a dropped row
|
||
# and a reconnect. This queue carries agent terminal rows
|
||
# published whole rather than split, so upstream's own
|
||
# default is sized to lose one of those rather than merely
|
||
# shorten it. Bounded from above by
|
||
# `max_pending`: nats-server refuses to start once this
|
||
# exceeds it, and widening that ceiling instead costs
|
||
# memory per connection, so this stays comfortably under it.
|
||
max_payload = 8388608;
|
||
|
||
# The monitoring endpoint, which is what the exporter below
|
||
# reads. `//` adds a key `calloutBlocks` does not produce
|
||
# (`accounts`, `authorization`) — checked, because a shallow
|
||
# merge that collided here would drop the auth config while
|
||
# rendering a config the server starts on.
|
||
#
|
||
# ⚠️ Loopback: unauthenticated, and `/connz` lists every
|
||
# connected client.
|
||
http = "127.0.0.1:${toString cfg.monitorPort}";
|
||
|
||
# TLS required on the client port: a server with a `tls`
|
||
# block advertises `tls_required` and refuses a client that
|
||
# does not upgrade. No `allow_non_tls`, which would keep the
|
||
# plaintext path open across the mesh. The 2s timeout is for
|
||
# a handshake that crosses the mesh; upstream's 0.5s is sized
|
||
# for a LAN.
|
||
tls = {
|
||
cert_file = tlsCertPath;
|
||
key_file = tlsKeyCredentialPath;
|
||
timeout = 2;
|
||
};
|
||
};
|
||
};
|
||
|
||
# The key as the `nats` user can read it; see `tlsDir`. Resolved at
|
||
# unit start only, which is why a renewed leaf restarts the server
|
||
# rather than reloading it.
|
||
systemd.services.nats.serviceConfig.LoadCredential = [
|
||
"${tlsKeyCredential}:${tlsKeyPath}"
|
||
];
|
||
|
||
# The translation layer. NATS has no Prometheus format of its
|
||
# own, so this reads the JSON monitoring endpoint above and
|
||
# re-serves it in the format the collector scrapes.
|
||
#
|
||
# Upstream's exporter module rather than a hand-rolled unit, for
|
||
# the same reason `services.nats`'s own `settings` is kept: the
|
||
# reviewed reasoning about flags and hardening lives there.
|
||
services.prometheus.exporters.nats = {
|
||
enable = true;
|
||
listenAddress = "127.0.0.1";
|
||
port = cfg.metricsPort;
|
||
url = "http://127.0.0.1:${toString cfg.monitorPort}";
|
||
|
||
# ⚠️ NOT optional, despite the name. Upstream renders
|
||
# `-addr … -port … ${extraFlags} ${url}` and defaults
|
||
# `extraFlags` to the empty list, but the exporter REFUSES TO
|
||
# START without at least one collector: it exits 1 with
|
||
# "no Collectors specified". Shipped without this the unit logs
|
||
# `Started`, the process is gone milliseconds later, and the
|
||
# scrape is refused — measured, and invisible to evaluation
|
||
# because the empty list renders perfectly.
|
||
#
|
||
# Which three, and why not more: `varz` is the server itself
|
||
# (uptime, memory, slow consumers), `connz` is per-connection so
|
||
# a client that will not stay connected is visible, and `jsz`
|
||
# covers JetStream, which this swarm relies on for the status
|
||
# KV. The remaining collectors describe a clustered deployment
|
||
# this swarm does not have.
|
||
extraFlags = [
|
||
"-varz"
|
||
"-connz"
|
||
"-jsz=all"
|
||
];
|
||
};
|
||
|
||
# A wrapper that includes upstream's rendered settings verbatim
|
||
# plus the runtime fragment; rendering the config ourselves
|
||
# instead would throw away upstream's `settings`, where the
|
||
# reviewed reasoning lives. The generator writes it, because the
|
||
# includes must be siblings of the fragment (see `runtimeDir`).
|
||
#
|
||
# `mkForce`: upstream defines ExecStart inside an `mkMerge`, so a
|
||
# plain override conflicts rather than wins.
|
||
systemd.services.nats.serviceConfig.ExecStart = lib.mkIf deployCfg.nats.autoGenerateCallout (
|
||
lib.mkForce "${pkgs.nats-server}/bin/nats-server -c ${runtimeWrapper}"
|
||
);
|
||
|
||
# The auth-callout responder: the half that lets the server
|
||
# above say *yes*. Without it the `auth_callout` block is a
|
||
# door nobody can open, which is the deliberate interim state.
|
||
#
|
||
# ⚠️ It is gated on the seeds being configured rather than on
|
||
# `deployCfg.nats.enable`, so a half-configured hive gets a running,
|
||
# refusing queue instead of a unit that crash-loops on a
|
||
# missing file. A queue that denies everyone is a legible
|
||
# failure; a restart loop is not.
|
||
systemd.services.swarm-nats-auth = lib.mkIf responderConfigured {
|
||
description = "swarm queue auth-callout responder";
|
||
after = [ "nats.service" ];
|
||
requires = [ "nats.service" ];
|
||
wantedBy = [ "multi-user.target" ];
|
||
serviceConfig = {
|
||
ExecStart = lib.concatStringsSep " " [
|
||
"${deployCfg.nats.authPackage}/bin/swarm-nats-auth"
|
||
# By name, like every other client: the leaf carries the name
|
||
# and no IP, so a loopback address fails verification. The
|
||
# container's resolver goes through the bridge, where dnsmasq
|
||
# answers it.
|
||
"--nats-url tls://${cfg.domain}:${toString cfg.port}"
|
||
"--user-seed-file \${CREDENTIALS_DIRECTORY}/callout-user.seed"
|
||
"--issuer-seed-file \${CREDENTIALS_DIRECTORY}/issuer.seed"
|
||
"--client-secret-file \${CREDENTIALS_DIRECTORY}/oidc-client.secret"
|
||
"--client-id ${lib.escapeShellArg cfg.clientId}"
|
||
# The account admitted clients land in, by NAME: in
|
||
# server-config mode the server resolves `aud` against its
|
||
# own `accounts` block, so this and the block above have to
|
||
# be the same string — which is why both come from one let.
|
||
"--account ${lib.escapeShellArg clientAccount}"
|
||
"--introspection-url ${lib.escapeShellArg introspectionUrl}"
|
||
# Each of these names a principal some OTHER module mints,
|
||
# so each is read out of that module rather than spelled
|
||
# again here — same argument as `--account` above, one
|
||
# level wider. The responder denies a client id it does
|
||
# not recognise, and a NATS denial arrives as a timeout,
|
||
# so a drift here is silent at the point of change and
|
||
# misattributed at the point of failure.
|
||
"--hive-client-prefix ${lib.escapeShellArg autheliaCfg.hiveClientPrefix}"
|
||
"--agent-client-suffix ${lib.escapeShellArg autheliaCfg.agentClientSuffix}"
|
||
"--reader-client ${lib.escapeShellArg controllerCfg.queueClientId}"
|
||
# What an agent may publish to, `{hive}` standing for the
|
||
# hive its client id names. Without it the responder has no
|
||
# agent grant to hand out and refuses every agent at CONNECT,
|
||
# so the terminal stream below depends on this line existing.
|
||
#
|
||
# `$$`, not `$`: systemd substitutes `$NAME` in `ExecStart`
|
||
# whether or not it is quoted, so a single dollar reaches the
|
||
# responder as the empty expansion of an unset `SWARM` and the
|
||
# grant silently becomes `.term.{hive}.>`.
|
||
"--agent-publish-subject ${lib.escapeShellArg "\$\$SWARM.term.{hive}.>"}"
|
||
# The turn-state header the swarm's agent view renders from
|
||
# (`hive-agent::swarm_agent_state`). Its own subject family
|
||
# rather than a leaf under `.term.`: a subscriber following
|
||
# one agent's terminal should not also be handed every
|
||
# header, and the two have opposite shapes — a terminal is
|
||
# an append-only row stream, a header is one current value
|
||
# republished on change.
|
||
"--agent-publish-subject ${lib.escapeShellArg "\$\$SWARM.agent-state.{hive}.>"}"
|
||
];
|
||
# Every credential arrives by `LoadCredential` and is named
|
||
# on the command line only as a **path** — `argv` is
|
||
# world-readable via /proc/<pid>/cmdline, so a value there
|
||
# would be readable by every process on the host netns.
|
||
LoadCredential = [
|
||
"callout-user.seed:${inContainer "callout-user.seed"}"
|
||
"issuer.seed:${inContainer "issuer.seed"}"
|
||
"oidc-client.secret:${inContainer "oidc-client.secret"}"
|
||
];
|
||
DynamicUser = true;
|
||
Restart = "on-failure";
|
||
RestartSec = "5s";
|
||
SyslogIdentifier = "swarm-nats-auth";
|
||
};
|
||
};
|
||
|
||
# The server binary, so an operator with a shell in here can
|
||
# run `nats-server -t` against the generated config. The unit
|
||
# resolves ExecStart through the store path and puts nothing
|
||
# on PATH.
|
||
environment.systemPackages = [ pkgs.nats-server ];
|
||
};
|
||
};
|
||
|
||
# Deliver the responder's three credentials into the container before
|
||
# it starts. Same shape as `hive-matrix-oidc-secret`, and for the same
|
||
# reason it is a copy rather than a `bindMounts` entry: nixos-container
|
||
# refuses to start when a bind source is missing, so one absent seed
|
||
# would take down the **whole container including the queue**, not
|
||
# merely the responder. A far larger blast radius than the fault.
|
||
# Order the container after the host CA generator, so the bind source
|
||
# exists before nspawn sets the mount up. Without it a late CA fails the
|
||
# container start outright rather than degrading.
|
||
systemd.services."container@${machine}" = caTrust.containerOrdering;
|
||
|
||
systemd.services.swarm-nats-auth-secrets = lib.mkIf responderConfigured {
|
||
description = "deliver the swarm queue responder's credentials";
|
||
before = [ "container@swarm-nats.service" ];
|
||
wantedBy = [ "container@swarm-nats.service" ];
|
||
# In auto mode the seeds this copies do not exist until the generator
|
||
# has run. `requires` as well as `after`: if minting fails there is
|
||
# nothing to deliver, and a copy that silently succeeds with a stale
|
||
# or absent seed is worse than not running.
|
||
after =
|
||
lib.optional deployCfg.nats.autoGenerateCallout "swarm-nats-callout-keys.service"
|
||
# The third credential does not come from the generator above — it is
|
||
# minted by authelia's FIRST BOOT, inside its own container. Ordering
|
||
# after that container is necessary and NOT sufficient: the container
|
||
# being up says nothing about whether its in-container secrets unit
|
||
# has finished. The wait in the script is what actually closes it;
|
||
# this only stops us spinning for the full timeout on every boot.
|
||
++ lib.optional deployCfg.authelia.enable "container@${autheliaCfg.machine}.service";
|
||
requires = lib.optional deployCfg.nats.autoGenerateCallout "swarm-nats-callout-keys.service";
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
# Must exceed the script's own wait below, and is set rather than
|
||
# left to the default for exactly that reason: systemd's
|
||
# `DefaultTimeoutStartSec` is 90s, so a 120s wait would be killed at
|
||
# 90 and the operator would get a generic unit timeout instead of
|
||
# the message that names the missing file and says what to do.
|
||
# The bound that matters belongs in one place, and this makes the
|
||
# two visibly related.
|
||
TimeoutStartSec = "180s";
|
||
SyslogIdentifier = "swarm-nats-auth-secrets";
|
||
};
|
||
path = [ pkgs.coreutils ];
|
||
script = ''
|
||
set -euo pipefail
|
||
install -d -m 0700 ${lib.escapeShellArg secretDir}
|
||
install -m 0400 ${lib.escapeShellArg userSeedFile} \
|
||
${lib.escapeShellArg (hostPath "callout-user.seed")}
|
||
install -m 0400 ${lib.escapeShellArg issuerSeedFile} \
|
||
${lib.escapeShellArg (hostPath "issuer.seed")}
|
||
# Wait for authelia's minted secret rather than failing the instant
|
||
# it is absent. On a fresh boot this unit and authelia's first-boot
|
||
# generator race, and losing that race used to cost the WHOLE QUEUE:
|
||
# this exits 1, the responder never starts, and `auth_callout` with
|
||
# no responder refuses every client — fail-closed by design, so the
|
||
# symptom appears on every queue client and nowhere near the cause.
|
||
#
|
||
# Bounded, not indefinite. Where authelia runs on another host the
|
||
# file is never going to appear, and blocking the queue container
|
||
# forever would replace a clear failure with a hang. After the
|
||
# timeout this fails exactly as it did before, having first given
|
||
# the co-located case the seconds it actually needs.
|
||
secret=${lib.escapeShellArg clientSecretSource}
|
||
deadline=$(( SECONDS + 120 ))
|
||
while [ ! -s "$secret" ]; do
|
||
if [ "$SECONDS" -ge "$deadline" ]; then
|
||
echo "the swarm queue responder's OIDC secret never appeared at $secret" >&2
|
||
echo "(authelia mints it on first boot; if authelia runs on another host," >&2
|
||
echo " copy the secret there and this unit will pick it up)" >&2
|
||
exit 1
|
||
fi
|
||
sleep 2
|
||
done
|
||
|
||
install -m 0400 "$secret" \
|
||
${lib.escapeShellArg (hostPath "oidc-client.secret")}
|
||
'';
|
||
};
|
||
|
||
# ⚠️ Minted on the HOST, not in the container, because the responder is
|
||
# a separate unit that needs the user seed: generating it inside would
|
||
# trap it there and require a secret-export path back out — the exact
|
||
# mechanism this is meant to avoid inventing. Only public halves cross.
|
||
systemd.services.swarm-nats-callout-keys = lib.mkIf deployCfg.nats.autoGenerateCallout {
|
||
description = "mint the swarm queue's auth-callout nkeys";
|
||
before = [ "container@swarm-nats.service" ];
|
||
wantedBy = [ "container@swarm-nats.service" ];
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
SyslogIdentifier = "swarm-nats-callout-keys";
|
||
};
|
||
path = [
|
||
pkgs.coreutils
|
||
pkgs.nkeys
|
||
];
|
||
script = ''
|
||
set -euo pipefail
|
||
umask 077
|
||
install -d -m 0700 ${lib.escapeShellArg autoSeedDir}
|
||
|
||
# Mint iff absent. Idempotence is the whole contract: this runs on
|
||
# every boot, and regenerating would silently invalidate every
|
||
# credential the responder has already issued against the old
|
||
# issuer.
|
||
if [ ! -s ${lib.escapeShellArg autoUserSeed} ]; then
|
||
nk -gen user > ${lib.escapeShellArg autoUserSeed}.tmp
|
||
mv ${lib.escapeShellArg autoUserSeed}.tmp ${lib.escapeShellArg autoUserSeed}
|
||
fi
|
||
if [ ! -s ${lib.escapeShellArg autoIssuerSeed} ]; then
|
||
nk -gen account > ${lib.escapeShellArg autoIssuerSeed}.tmp
|
||
mv ${lib.escapeShellArg autoIssuerSeed}.tmp ${lib.escapeShellArg autoIssuerSeed}
|
||
fi
|
||
chmod 0600 ${lib.escapeShellArg autoUserSeed} ${lib.escapeShellArg autoIssuerSeed}
|
||
|
||
user_pub="$(nk -inkey ${lib.escapeShellArg autoUserSeed} -pubout)"
|
||
issuer_pub="$(nk -inkey ${lib.escapeShellArg autoIssuerSeed} -pubout)"
|
||
|
||
# Public halves only — world-readable on purpose, since the server
|
||
# publishes them to every client that connects.
|
||
install -d -m 0755 ${lib.escapeShellArg hostRuntimeDir}
|
||
sed -e "s|@USER_PUBKEY@|$user_pub|g" \
|
||
-e "s|@ISSUER_PUBKEY@|$issuer_pub|g" \
|
||
${calloutTemplate} > ${lib.escapeShellArg hostRuntimeDir}/callout.conf.tmp
|
||
# Mode BEFORE the rename: a rename publishes whatever the file
|
||
# already is, so setting it afterwards leaves a window where the
|
||
# live path has the wrong mode. Same trap as the gateway's
|
||
# atomic-publish path.
|
||
chmod 0444 ${lib.escapeShellArg hostRuntimeDir}/callout.conf.tmp
|
||
mv ${lib.escapeShellArg hostRuntimeDir}/callout.conf.tmp \
|
||
${lib.escapeShellArg hostRuntimeDir}/callout.conf
|
||
|
||
# Upstream's rendered settings, as a sibling the wrapper can name
|
||
# without a leading slash. Refreshed unconditionally — unlike the
|
||
# seeds, this one MUST track the current system, and a stale copy
|
||
# would silently run yesterday's config.
|
||
ln -sfn ${renderedSettings} ${lib.escapeShellArg hostRuntimeDir}/settings.conf
|
||
|
||
# The wrapper. Bare filenames — see `runtimeDir`. printf rather than
|
||
# a heredoc, whose terminator would depend on nix's indentation
|
||
# stripping and break the next time `nix fmt` touched this block.
|
||
printf 'include "settings.conf"\ninclude "callout.conf"\n' \
|
||
> ${lib.escapeShellArg hostRuntimeDir}/nats.conf.tmp
|
||
chmod 0444 ${lib.escapeShellArg hostRuntimeDir}/nats.conf.tmp
|
||
mv ${lib.escapeShellArg hostRuntimeDir}/nats.conf.tmp \
|
||
${lib.escapeShellArg hostRuntimeDir}/nats.conf
|
||
'';
|
||
};
|
||
|
||
# The queue's TLS leaf, issued by the secret store's `pki/roles/swarm-nats`
|
||
# under this host's `swarm-nats` identity. Same shape as ./hive-tls.nix's
|
||
# `swarm-services-cert`, whose comments carry the reasoning for the login,
|
||
# the single `issue` call split with `jq`, and the unseal-sized retry.
|
||
#
|
||
# Before the container, so a normal boot starts the server with its leaf.
|
||
# A store that comes up later fails this run; the retry lands the leaf and
|
||
# restarts the server, which until then refuses to start (TLS required,
|
||
# and its key credential is missing).
|
||
systemd.services.swarm-bao-nats-tls = {
|
||
description = "Issue the swarm queue's TLS leaf from the secret store's PKI";
|
||
wantedBy = [
|
||
"multi-user.target"
|
||
"container@${machine}.service"
|
||
];
|
||
before = [ "container@${machine}.service" ];
|
||
# Both absent where the store runs elsewhere, and ignored there; see
|
||
# `swarm-services-cert`. The ordering after this leaf's policy unit is
|
||
# ./glue-bao-readers-policy-order.nix's, because it holds only where the
|
||
# store is here too.
|
||
after = [
|
||
"container@${baoCfg.machine}.service"
|
||
"swarm-bao-pki.service"
|
||
];
|
||
wants = [ "container@${baoCfg.machine}.service" ];
|
||
path = [
|
||
baoDeploy.package
|
||
pkgs.jq
|
||
pkgs.openssl
|
||
pkgs.coreutils
|
||
pkgs.gnugrep
|
||
pkgs.systemd
|
||
];
|
||
startLimitBurst = 2880;
|
||
startLimitIntervalSec = 90000;
|
||
# No `RemainAfterExit`: the timer below starts this again, and starting
|
||
# an active unit is a no-op.
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
UMask = "0077";
|
||
SyslogIdentifier = "swarm-bao-nats-tls";
|
||
Restart = "on-failure";
|
||
RestartSec = 30;
|
||
};
|
||
environment = {
|
||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||
}
|
||
// lib.optionalAttrs (deployCfg.nats.baoClientCertFile != null) {
|
||
BAO_CLIENT_CERT = deployCfg.nats.baoClientCertFile;
|
||
}
|
||
// lib.optionalAttrs (deployCfg.nats.baoClientKeyFile != null) {
|
||
BAO_CLIENT_KEY = deployCfg.nats.baoClientKeyFile;
|
||
}
|
||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||
BAO_CACERT = baoDeploy.serverCaFile;
|
||
};
|
||
script = ''
|
||
set -euo pipefail
|
||
d=${lib.escapeShellArg tlsDir}
|
||
name=${lib.escapeShellArg cfg.domain}
|
||
install -d -m 0755 "$d"
|
||
|
||
# Missing, past half its window, or naming something other than the
|
||
# configured domain.
|
||
reissue=0
|
||
{ [ -s "$d/cert.pem" ] && [ -s "$d/key.pem" ]; } || reissue=1
|
||
openssl x509 -in "$d/cert.pem" -noout -checkend ${toString leafRenewSeconds} >/dev/null 2>&1 || reissue=1
|
||
openssl x509 -in "$d/cert.pem" -noout -checkhost "$name" 2>/dev/null | grep -q ' does match ' || reissue=1
|
||
|
||
if [ "$reissue" = 0 ]; then
|
||
echo "queue leaf valid for $name — leaving it alone"
|
||
exit 0
|
||
fi
|
||
|
||
${
|
||
if deployCfg.nats.baoClientCertFile == null || deployCfg.nats.baoClientKeyFile == null then
|
||
''
|
||
echo "no bao client certificate configured for the queue, so its TLS leaf" >&2
|
||
echo "cannot be requested from the store." >&2
|
||
echo "Set services.hyperhive.deploy.nats.baoClient{Cert,Key}File" >&2
|
||
echo "to a leaf the store's CA signed with CN=${baoDeploy.natsCommonName}." >&2
|
||
exit 1''
|
||
else
|
||
""
|
||
}
|
||
|
||
err="$(mktemp)"
|
||
trap 'rm -f "$err"' EXIT
|
||
|
||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||
echo "could not log in to the swarm secret store with the queue's certificate." >&2
|
||
cat "$err" >&2
|
||
exit 1
|
||
fi
|
||
export BAO_TOKEN
|
||
|
||
echo "requesting the queue's TLS leaf for $name from the store"
|
||
resp="$(mktemp "$d/issue.json.XXXXXX")"
|
||
trap 'rm -f "$err" "$resp"' EXIT
|
||
|
||
if ! bao write -format=json \
|
||
${lib.escapeShellArg "${baoDeploy.servicesPkiMountPath}/issue/${baoDeploy.natsPkiRoleName}"} \
|
||
common_name="$name" > "$resp" 2>"$err"; then
|
||
echo "the store refused to issue the queue's certificate." >&2
|
||
cat "$err" >&2
|
||
exit 1
|
||
fi
|
||
|
||
jq -r '.data.private_key // empty' < "$resp" > "$d/key.pem.new"
|
||
jq -r '.data.certificate // empty' < "$resp" > "$d/cert.pem.new"
|
||
for f in "$d/key.pem.new" "$d/cert.pem.new"; do
|
||
if [ ! -s "$f" ]; then
|
||
echo "the store's response was missing a field: $f is empty" >&2
|
||
exit 1
|
||
fi
|
||
done
|
||
chmod 0600 "$d/key.pem.new"
|
||
chmod 0644 "$d/cert.pem.new"
|
||
mv -f "$d/key.pem.new" "$d/key.pem"
|
||
mv -f "$d/cert.pem.new" "$d/cert.pem"
|
||
|
||
# Renewal, or the late-store retry. On a normal boot the container is
|
||
# not up yet and starts the server with this leaf itself. A restart,
|
||
# not a reload: the key arrives by `LoadCredential`, which systemd
|
||
# resolves at start only. `reset-failed` because a server that found
|
||
# no key may have hit its start limit.
|
||
if systemctl is-active --quiet ${lib.escapeShellArg "container@${machine}.service"}; then
|
||
echo "queue leaf rotated — restarting the server in ${machine}"
|
||
systemctl --machine=${lib.escapeShellArg machine} reset-failed nats.service
|
||
systemctl --machine=${lib.escapeShellArg machine} restart nats.service
|
||
fi
|
||
'';
|
||
};
|
||
|
||
systemd.timers.swarm-bao-nats-tls = {
|
||
description = "Daily renewal check for the swarm queue's TLS leaf";
|
||
wantedBy = [ "timers.target" ];
|
||
timerConfig = {
|
||
OnCalendar = "daily";
|
||
Persistent = true;
|
||
};
|
||
};
|
||
};
|
||
}
|