b5053d9a80
`swarm-bao-pki` was a `RemainAfterExit` oneshot. Once active, nothing re-ran it: a switch restarts it only when its definition changes, and a `Requires=`/`Wants=` on an active unit is satisfied without running it. On muede-lpt2 the 10:57Z switch that added the builder line did change the unit, so switch-to-configuration stopped it in its stop phase. A start of `container@swarm-bao` 4 s later (before the daemon-reload at 10:57:35Z) pulled it back in through `swarm-bao-certs` with the OLD definition, which had no builder line: 48 ms, nothing missing. The start phase then found it active and did nothing. The 20:14Z switch that enabled the builder left the unit unchanged, so it was never re-run, and `container@swarm-builder` crash-looped on the absent `builder.pem`. The unit now goes inactive after each run. Every switch starts `multi-user.target`, which runs it again post-reload with the current definition, and every unit that wants or requires it runs it on its own start. Each line is idempotent on presence, so an existing leaf is never re-signed. The three consumers started from `machines.target` rather than `multi-user.target` also want it, since `After=` alone does not order against a job in another transaction: the builder's and the review classifier's containers (nspawn `--load-credential`) and `swarm-nats-auth-secrets` (pulled in by the queue's container). Fixes #5284
1152 lines
53 KiB
Nix
1152 lines
53 KiB
Nix
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
cfg = config.services.hyperhive.swarm.nats;
|
||
autheliaCfg = config.services.hyperhive.swarm.authelia;
|
||
deployCfg = config.services.hyperhive.deploy;
|
||
autheliaUrl = autheliaCfg.url;
|
||
networkCfg = config.services.hyperhive.network;
|
||
# Read even when the controller runs on a different host: what is needed
|
||
# is the client id that module *declares*, which is the same string
|
||
# everywhere, not whether the daemon happens to be enabled here.
|
||
controllerCfg = config.services.hyperhive.swarm.controller;
|
||
|
||
# The account the callout responder authenticates as, and the account
|
||
# authorized clients are placed in. Two accounts rather than one: an
|
||
# account is NATS' isolation boundary, so a responder that shares an
|
||
# account with its clients can be published to by the things it
|
||
# authorizes.
|
||
calloutAccount = "AUTH";
|
||
clientAccount = "APP";
|
||
|
||
machine = "swarm-nats";
|
||
|
||
tlsCfg = config.services.hyperhive.deploy.hive-controller.tls;
|
||
gatewayCfg = config.services.hyperhive.gateway;
|
||
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
||
# The responder introspects authelia over https BY NAME. Its HTTP client is
|
||
# reqwest/rustls, and `rustls-platform-verifier` resolves roots through
|
||
# `rustls-native-certs`, which reads `SSL_CERT_FILE` — so the same assembled
|
||
# bundle the Go containers use applies here. Without it the handshake fails
|
||
# `UnknownIssuer`, introspection fails, and the responder denies *every*
|
||
# client: one missing trust anchor surfacing as `authorization violation` at
|
||
# every would-be queue user.
|
||
caBundleModule = caTrust.trustBundle {
|
||
inherit pkgs;
|
||
name = machine;
|
||
consumers = [ "swarm-nats-auth" ];
|
||
};
|
||
# Where the responder's credentials live *inside* the container, and the
|
||
# host path that resolves to. Two names for one location, because the
|
||
# host is the only place both filesystems are addressable.
|
||
secretDirInContainer = "/var/lib/swarm-nats-auth";
|
||
inContainer = name: "${secretDirInContainer}/${name}";
|
||
secretDir = "/var/lib/nixos-containers/${machine}${secretDirInContainer}";
|
||
hostPath = name: "${secretDir}/${name}";
|
||
|
||
introspectionUrl = "${toString autheliaUrl}/api/oidc/introspection";
|
||
|
||
# Host seed files `ensure-callout-keys` stores when the secret store holds
|
||
# no seed for that key, and removes once it holds one. Never written here.
|
||
importSeedDir = "/var/lib/swarm-nats-callout";
|
||
importUserSeed = "${importSeedDir}/callout-user.seed";
|
||
importIssuerSeed = "${importSeedDir}/issuer.seed";
|
||
|
||
# The runtime config directory: wrapper, settings symlink and fragment.
|
||
# World-readable is correct (everything in it is public) and it is not in
|
||
# the 0700 responder secret dir, which the `nats` user cannot traverse.
|
||
#
|
||
# ⚠️ ONE DIRECTORY IS FORCED, NOT TIDINESS. NATS resolves an `include` with
|
||
# `filepath.Join(configDir, path)`, which strips a leading slash, so an
|
||
# absolute include silently becomes relative and is never found. The
|
||
# includes must therefore be bare filenames, i.e. siblings. Invisible to
|
||
# eval: the wrapper renders perfectly and the server refuses to start.
|
||
runtimeDir = "/var/lib/nats-callout";
|
||
runtimeWrapper = "${runtimeDir}/nats.conf";
|
||
hostRuntimeDir = "/var/lib/nixos-containers/${machine}${runtimeDir}";
|
||
|
||
natsFormat = pkgs.formats.json { };
|
||
|
||
# Upstream's rendered settings, re-rendered with the same generator on the
|
||
# same value — the artifact `services.nats` would have used, not a
|
||
# transcription. ⚠️ `renderConfig` below is the only thing naming it:
|
||
# `ExecStart` names a runtime path, so that reference is what keeps the store
|
||
# file in the closure and stops it being garbage-collected out from under a
|
||
# running server.
|
||
renderedSettings = natsFormat.generate "nats.conf" (
|
||
config.containers.${machine}.config.services.nats.settings
|
||
);
|
||
|
||
# Defined once, rendered twice: into `settings` with no keys, and into the
|
||
# runtime fragment with placeholders for the three public keys, filled in
|
||
# at server start.
|
||
#
|
||
# 🪤 Do not inline these and hand-write the fragment instead. Both files are
|
||
# loaded and the fragment is the LATER definition, so it wins — a
|
||
# hand-written copy would make a future edit to `settings` silently
|
||
# ineffective.
|
||
calloutBlocks =
|
||
{
|
||
userKey,
|
||
issuerKey,
|
||
xkey ? null,
|
||
}:
|
||
{
|
||
# Two accounts, and the callout user lives in neither of the accounts
|
||
# it authorizes into.
|
||
accounts = {
|
||
# ⚠️ The nkey is not decoration, and without it there is a hole: a
|
||
# `users` entry carrying only a `user` name has no credential, and
|
||
# `CONNECT {"user":"auth"}` is then accepted with no password at
|
||
# all. Since the name is a literal in this public module, that made
|
||
# the callout-exempt identity walk-in-able from every container on
|
||
# the shared netns — the same class of hole this module exists to
|
||
# close. An nkey and NOTHING else, both halves
|
||
# measured against a running server rather than reasoned about:
|
||
#
|
||
# { user = "auth"; } → `CONNECT {"user":"auth"}`
|
||
# is accepted with no
|
||
# credential at all
|
||
# { user = "auth"; nkey = "U…"; } → refuses to START:
|
||
# "Nkey users do not take
|
||
# usernames or passwords"
|
||
# { nkey = "U…"; } → what this is
|
||
#
|
||
# A malformed key is fail-closed too: the server exits with
|
||
# "Not a valid public nkey for a user" rather than starting with a
|
||
# hole. So the only way to get a live server here is a real key
|
||
# whose seed nobody but the responder holds.
|
||
#
|
||
# 🔒 That property is what makes `settings` safe: both nkeys render
|
||
# as "" there, so any field the fragment fails to override keeps a
|
||
# value the server refuses to start on. An incomplete merge cannot leave
|
||
# a walk-in-able server.
|
||
${calloutAccount}.users = [ { nkey = userKey; } ];
|
||
# ⚠️ `services.nats.jetstream = true` gives the SERVER JetStream;
|
||
# an account gets it only from its own grant. Measured against a
|
||
# running 2.14.1 with this exact two-account shape, because the
|
||
# failure is invisible to any config-rendering check:
|
||
#
|
||
# global jetstream only → `nats kv add` from this account fails
|
||
# `code=503 err_code=10039 jetstream not enabled for account`,
|
||
# while the server starts cleanly and logs "Starting JetStream"
|
||
# + this line → the same command succeeds
|
||
#
|
||
# The grant is per-account by design, and that is worth keeping:
|
||
# the callout account above deliberately does NOT get it. The
|
||
# responder mints credentials; it has no business holding stream
|
||
# state.
|
||
#
|
||
# ⚠️ Also why the fragment renders the COMPLETE accounts block: if a
|
||
# later definition replaced rather than merged, a partial one would
|
||
# drop this grant and every KV op would fail on a healthy server.
|
||
${clientAccount} = {
|
||
jetstream = "enabled";
|
||
};
|
||
};
|
||
|
||
authorization = {
|
||
timeout = "2s";
|
||
# 🔒 THIS BLOCK IS THE FAIL-CLOSED STATE, and it is the measured
|
||
# one rather than the obvious one.
|
||
#
|
||
# Measured on the pinned nats-server 2.14.1: both
|
||
# `authorization { }` and `authorization { users: [] }` accept an
|
||
# anonymous client and answer PONG — they read like "authorize
|
||
# nobody" and are wide open. An auth_callout block sets
|
||
# `auth_required` and refuses every client whose credential no
|
||
# responder has approved, so a config whose responder does not
|
||
# exist yet denies everyone.
|
||
#
|
||
# `nats-server -t` calls all three valid; it parses, it does not
|
||
# authenticate. Only running them tells the difference.
|
||
#
|
||
# ⇒ this is both the safe interim state and the final shape.
|
||
# Nothing here has to be swapped out when the responder lands
|
||
# beside it — it only starts being able to say yes.
|
||
auth_callout = {
|
||
issuer = issuerKey;
|
||
auth_users = [ userKey ];
|
||
account = calloutAccount;
|
||
}
|
||
# The curve key the server seals each request to, and opens the
|
||
# reply with. A key that does not parse stops the server starting.
|
||
// lib.optionalAttrs (xkey != null) { inherit xkey; };
|
||
};
|
||
};
|
||
|
||
# The callout's three public keys: where the store keeps each, the file
|
||
# `swarm-bao-nats-callout-xkey` leaves it in, the placeholder it fills in
|
||
# `calloutTemplate`, and the shape `renderConfig` checks it has. The store
|
||
# paths are `swarm-nats-auth`'s `xkey::PUBLIC_PATH` and the `public_path` of
|
||
# its `nkey::ISSUER` and `nkey::USER`, readable by the `swarm-nats-auth` role
|
||
# alone; each seed beside them is in ./swarm-bao.nix.
|
||
calloutPublicKeys = [
|
||
{
|
||
storePath = "secret/swarm/public/swarm-nats/callout-xkey";
|
||
file = "callout-xkey.pub";
|
||
placeholder = "@XKEY_PUBKEY@";
|
||
var = "xkey";
|
||
pattern = "X[A-Z2-7]{55}";
|
||
kind = "public xkey";
|
||
}
|
||
{
|
||
storePath = "secret/swarm/public/swarm-nats/callout-issuer";
|
||
file = "issuer.pub";
|
||
placeholder = "@ISSUER_PUBKEY@";
|
||
var = "issuer_pub";
|
||
pattern = "A[A-Z2-7]{55}";
|
||
kind = "public account nkey";
|
||
}
|
||
{
|
||
storePath = "secret/swarm/public/swarm-nats/callout-user";
|
||
file = "callout-user.pub";
|
||
placeholder = "@USER_PUBKEY@";
|
||
var = "user_pub";
|
||
pattern = "U[A-Z2-7]{55}";
|
||
kind = "public user nkey";
|
||
}
|
||
];
|
||
|
||
# The fragment as nix renders it, with placeholders where the runtime
|
||
# values go. Rendered by the same JSON generator upstream uses, so the
|
||
# fragment is generated rather than transcribed — NATS' config parser
|
||
# accepts JSON, and an `include` of it merges (measured).
|
||
calloutTemplate = natsFormat.generate "swarm-nats-callout-template.conf" (calloutBlocks {
|
||
userKey = "@USER_PUBKEY@";
|
||
issuerKey = "@ISSUER_PUBKEY@";
|
||
xkey = "@XKEY_PUBKEY@";
|
||
});
|
||
|
||
# Each placeholder in `calloutTemplate`, as a `sed` expression over the
|
||
# shell variable `renderConfig` reads its value into.
|
||
calloutSubstitutions = map (k: ''-e "s|${k.placeholder}|${"$"}${k.var}|g"'') calloutPublicKeys;
|
||
|
||
# Renders the server's config as it starts, so the settings it includes are
|
||
# always the running system's. Exits non-zero, and so stops the server
|
||
# starting, while any of the three public keys has not been fetched: the
|
||
# server never runs without them. `+` because the server's own sandbox
|
||
# cannot write `runtimeDir`.
|
||
renderConfig = pkgs.writeShellScript "swarm-nats-render-config" ''
|
||
set -euo pipefail
|
||
PATH=${
|
||
lib.makeBinPath [
|
||
pkgs.coreutils
|
||
pkgs.gnugrep
|
||
pkgs.gnused
|
||
]
|
||
}
|
||
d=${lib.escapeShellArg runtimeDir}
|
||
|
||
${lib.concatMapStrings (k: ''
|
||
if [ ! -s "$d/${k.file}" ]; then
|
||
echo "${runtimeDir}/${k.file} is missing: swarm-bao-nats-callout-xkey on the host fetches it from the secret store, and the queue does not start without it" >&2
|
||
exit 1
|
||
fi
|
||
${k.var}="$(cat "$d/${k.file}")"
|
||
# The key type's letter and 55 base32 characters; checked here because
|
||
# it is about to become part of a sed expression.
|
||
if ! printf '%s' "${"$"}${k.var}" | grep -Eqx '${k.pattern}'; then
|
||
echo "${runtimeDir}/${k.file} does not hold a ${k.kind}" >&2
|
||
exit 1
|
||
fi
|
||
'') calloutPublicKeys}
|
||
sed ${lib.concatStringsSep " " calloutSubstitutions} ${calloutTemplate} > "$d/callout.conf.tmp"
|
||
# Mode BEFORE the rename: a rename publishes whatever the file already
|
||
# is. World-readable, because the server reads it as `nats` and
|
||
# everything in it is public.
|
||
chmod 0444 "$d/callout.conf.tmp"
|
||
mv "$d/callout.conf.tmp" "$d/callout.conf"
|
||
|
||
# Upstream's rendered settings, as a sibling the wrapper can name without
|
||
# a leading slash.
|
||
ln -sfn ${renderedSettings} "$d/settings.conf"
|
||
|
||
# The wrapper. Bare filenames — see `runtimeDir`. printf rather than a
|
||
# heredoc, whose terminator would depend on nix's indentation stripping.
|
||
printf 'include "settings.conf"\ninclude "callout.conf"\n' > "$d/nats.conf.tmp"
|
||
chmod 0444 "$d/nats.conf.tmp"
|
||
mv "$d/nats.conf.tmp" "$d/nats.conf"
|
||
'';
|
||
|
||
baoCfg = config.services.hyperhive.swarm.bao;
|
||
baoDeploy = deployCfg.bao;
|
||
|
||
# The queue's TLS leaf, bound read-only at the same path on both sides, the
|
||
# shape ./swarm-bao.nix's `tlsDir` uses. The key reaches the server through
|
||
# `LoadCredential`, as openbao's does: it is root-owned 0600 here, and the
|
||
# server runs as `nats`.
|
||
tlsDir = "/var/lib/swarm-nats-tls";
|
||
tlsCertPath = "${tlsDir}/cert.pem";
|
||
tlsKeyPath = "${tlsDir}/key.pem";
|
||
tlsKeyCredential = "tls-key";
|
||
tlsKeyCredentialPath = "/run/credentials/nats.service/${tlsKeyCredential}";
|
||
|
||
# The gateway's copy of that leaf, for the `/metrics` vhost below: the key
|
||
# here is 0640 root:nginx, so nginx reads it while the queue's own stays
|
||
# 0600 root. Self-signed mode only, the mode `gateway.lib.tlsFor` hands the
|
||
# swarm-services leaf to; with ACME or an operator cert that issuer covers
|
||
# this name as it covers every other.
|
||
gatewayServesQueueLeaf = gatewayCfg.enable && gatewayCfg.useSelfSigned;
|
||
gatewayTlsDir = "/var/lib/swarm-nats-gateway-tls";
|
||
|
||
# The responder's own store identity, for the callout xkey, agent queue
|
||
# credentials and its OIDC client secret. Required: an assertion below refuses
|
||
# a queue without it.
|
||
authStoreActive =
|
||
deployCfg.nats.authBaoClientCertFile != null && deployCfg.nats.authBaoClientKeyFile != null;
|
||
# The role ./swarm-bao.nix writes on the store's `cert` mount, by the same name.
|
||
authCertRole = "swarm-nats-auth";
|
||
# Store identity files the copy unit delivers, as credential id → host source.
|
||
authStoreFiles = {
|
||
"bao-client.pem" = deployCfg.nats.authBaoClientCertFile;
|
||
"bao-client-key.pem" = deployCfg.nats.authBaoClientKeyFile;
|
||
}
|
||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||
"bao-ca.pem" = baoDeploy.serverCaFile;
|
||
};
|
||
authCredential = id: "/run/credentials/swarm-nats-auth.service/${id}";
|
||
|
||
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
|
||
refreshConsumer = import ./lib/refresh-consumer.nix { };
|
||
storeRetry = import ./lib/store-retry.nix { };
|
||
|
||
# Re-issue once the leaf is past half of the 720h `pki/roles/swarm-nats`
|
||
# grants it (./swarm-bao.nix), so the daily timer below has two weeks of
|
||
# retries before it lapses.
|
||
leafRenewSeconds = 360 * 3600;
|
||
|
||
# Shared host netns, like every sibling swarm container.
|
||
#
|
||
# ⚠️ Which is exactly why the server below must refuse everyone
|
||
# until the callout responder exists: the queue is on the host's
|
||
# own loopback, in reach of every process there and every sibling
|
||
# on this netns, and each remote hive in a multi-host swarm dials
|
||
# it directly. An unauthenticated interim state would be a hole
|
||
# rather than a rough edge.
|
||
#
|
||
# Not agent containers, though: they have a netns of their own and
|
||
# the bridge firewall does not open this port.
|
||
privateNetwork = false;
|
||
in
|
||
{
|
||
# The swarm's message queue: one NATS server, reached by every hive at
|
||
# `tls://<swarm.nats.domain>:<port>`.
|
||
#
|
||
# ⚠️ The gateway does NOT front the queue. NATS speaks its own TCP protocol
|
||
# rather than HTTP, so the server terminates TLS itself on `port`. The
|
||
# name is the half of the sibling pattern that applies, the way it is for
|
||
# bao: `gateway.localNames` on this host, the operator's DNS everywhere
|
||
# else. The one gateway vhost on that name serves the exporter's
|
||
# `/metrics` to the swarm collector and nothing else.
|
||
|
||
# What the queue IS to every hive, the ports it answers on and the client id
|
||
# it is registered under, is `swarm.nats` in ./swarm-nats-service.nix. What
|
||
# the host running it decides is below — which responder build it runs and
|
||
# the store identities its units present. `enable` already lives in
|
||
# ./deploy.nix, which also carries the renames. The callout's keys are no
|
||
# option at all: they live in the secret store (docs/swarm/secrets.md).
|
||
options.services.hyperhive.deploy.nats = {
|
||
authPackage = lib.mkOption {
|
||
type = lib.types.package;
|
||
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-nats-auth";
|
||
description = ''
|
||
The auth-callout responder package.
|
||
|
||
⚠️ Named `authPackage`, not `package`, on purpose: this module
|
||
deliberately has **no** `package` option for the NATS server
|
||
itself, because upstream's `services.nats` resolves
|
||
`pkgs.nats-server` on its own — see the note above
|
||
`options.services.hyperhive.swarm.nats`. A bare `package` here
|
||
would read as "the NATS package" and mean something else
|
||
entirely.
|
||
'';
|
||
};
|
||
|
||
baoClientCertFile = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/var/lib/swarm-bao-pki/nats.pem";
|
||
description = ''
|
||
Client certificate `swarm-bao-nats-tls` presents to the swarm's
|
||
secret store when it asks for the queue's TLS leaf. Its subject must
|
||
be {option}`services.hyperhive.deploy.bao.natsCommonName`: cert auth
|
||
matches on the CN, and that role's policy may issue from
|
||
`pki/issue/swarm-nats` and nothing else.
|
||
|
||
No default. ./glue-nats-bao-identity.nix points it at the leaf
|
||
./glue-bao-tls.nix mints on a host that runs the store
|
||
({option}`services.hyperhive.deploy.bao.enable`). On a queue host
|
||
without the store, it is a file the operator copies from the store's
|
||
host. Never derived from
|
||
{option}`services.hyperhive.deploy.bao.clientCertFile`, which is the
|
||
hive's own reader.
|
||
|
||
A path, never a value.
|
||
'';
|
||
};
|
||
|
||
baoClientKeyFile = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/var/lib/swarm-bao-pki/nats-key.pem";
|
||
description = ''
|
||
Private key for {option}`services.hyperhive.deploy.nats.baoClientCertFile`.
|
||
A path, never a value.
|
||
'';
|
||
};
|
||
|
||
authBaoClientCertFile = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/var/lib/swarm-bao-pki/nats-auth.pem";
|
||
description = ''
|
||
Client certificate the auth-callout responder presents to the swarm's
|
||
secret store to read the callout keys, its own OIDC client secret and
|
||
agent queue credentials, and that `swarm-bao-nats-callout-xkey`
|
||
presents to read the callout keys' public halves. Its subject must be
|
||
{option}`services.hyperhive.deploy.bao.natsAuthCommonName`.
|
||
|
||
Required when `deploy.nats.enable` is set; unset fails at eval.
|
||
./glue-nats-auth-bao-identity.nix points it at the leaf
|
||
./glue-bao-tls.nix mints on a host that runs the store
|
||
({option}`services.hyperhive.deploy.bao.enable`); on any other queue
|
||
host it is a file the operator places. Never derived from
|
||
{option}`services.hyperhive.deploy.bao.clientCertFile`, which is the
|
||
hive's own reader. Set to a file that does
|
||
not exist, `swarm-nats-auth-secrets` and `swarm-bao-nats-callout-xkey`
|
||
fail, and the queue's server does not start without the xkey.
|
||
|
||
A path, never a value.
|
||
'';
|
||
};
|
||
|
||
authBaoClientKeyFile = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/var/lib/swarm-bao-pki/nats-auth-key.pem";
|
||
description = ''
|
||
Private key for {option}`services.hyperhive.deploy.nats.authBaoClientCertFile`.
|
||
A path, never a value.
|
||
'';
|
||
};
|
||
};
|
||
|
||
config = lib.mkIf deployCfg.nats.enable {
|
||
# The auth responder introspects authelia by name, and the gateway serves
|
||
# the exporter's `/metrics` below.
|
||
services.hyperhive.gateway.enable = lib.mkDefault true;
|
||
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
||
|
||
# The name every client dials, answered with the bridge IP on this host.
|
||
# Other hives resolve it through the operator's DNS, as they do bao's.
|
||
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
||
|
||
# The exporter's metrics, for the swarm collector wherever it runs, and
|
||
# nothing else on this name: the monitoring endpoint (`/varz`, `/connz`)
|
||
# is never proxied, and every other path is 403.
|
||
#
|
||
# ⚠️ Not `gateway.lib.tlsFor`, and the name is not in
|
||
# `swarm.serviceDomains`. In self-signed mode this vhost serves the
|
||
# queue's own leaf (issued for exactly this name, `server_flag` set), so
|
||
# a host whose only swarm service is the queue never fronts a services
|
||
# name and its gateway does not wait for the swarm-services leaf.
|
||
#
|
||
# `forceSSL` for the reason ./swarm-victorialogs.nix gives: authelia
|
||
# answers a plain-http auth subrequest with 400.
|
||
services.nginx.virtualHosts."${cfg.domain}" =
|
||
(
|
||
if gatewayServesQueueLeaf then
|
||
{
|
||
sslCertificate = "${gatewayTlsDir}/cert.pem";
|
||
sslCertificateKey = "${gatewayTlsDir}/key.pem";
|
||
}
|
||
else
|
||
builtins.removeAttrs (gatewayCfg.lib.tlsFor cfg.domain) [ "addSSL" ]
|
||
)
|
||
// {
|
||
forceSSL = true;
|
||
listen = gatewayCfg.lib.listen;
|
||
extraConfig = gatewayCfg.lib.securityHeaders;
|
||
locations = {
|
||
"/".return = "403";
|
||
|
||
# The collector requests exactly this URL, declared as
|
||
# `publishedScrapeTargets.nats` in ./swarm-otel-service.nix; the
|
||
# two must agree, since that string is also the token's audience.
|
||
# Authelia's rules for it admit the collector's client and deny
|
||
# every other subject (`metricsRules` in ./swarm-authelia.nix).
|
||
"= /metrics" = {
|
||
proxyPass = "http://127.0.0.1:${toString cfg.metricsPort}/metrics";
|
||
extraConfig = ''
|
||
auth_request /__nats_metrics_authz;
|
||
'';
|
||
};
|
||
|
||
# The subrequest, same shape as the forge's `= /__forge_metrics_authz`
|
||
# (./hive-forge/default.nix): no `error_page 401 =302`, so a scraper
|
||
# gets the 401 rather than a login page parsed as metrics.
|
||
"= /__nats_metrics_authz" = {
|
||
proxyPass = "https://${autheliaCfg.domain}/api/authz/auth-request";
|
||
# nixpkgs appends its OWN `Host $host` after extraConfig, which
|
||
# would override verifiedProxyTo's — see hive-gateway/vhost-lib.nix.
|
||
recommendedProxySettings = false;
|
||
extraConfig = ''
|
||
internal;
|
||
${gatewayCfg.lib.verifiedProxyTo autheliaCfg.domain}
|
||
proxy_pass_request_body off;
|
||
proxy_set_header Content-Length "";
|
||
proxy_set_header X-Original-Method $request_method;
|
||
proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
|
||
proxy_set_header X-Forwarded-Proto $scheme;
|
||
proxy_set_header X-Forwarded-Host $http_host;
|
||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||
'';
|
||
};
|
||
};
|
||
};
|
||
|
||
# Puts a pair at the paths the vhost above names, before nginx starts:
|
||
# nginx refuses a config naming a missing cert file, and that takes the
|
||
# whole gateway down rather than this one name. The queue's leaf when it
|
||
# exists, else a throwaway self-signed pair until `swarm-bao-nats-tls`
|
||
# issues the leaf and starts this again. Never fails on a missing leaf.
|
||
systemd.services.swarm-nats-gateway-cert = lib.mkIf gatewayServesQueueLeaf {
|
||
description = "Copy the swarm queue's TLS leaf for the gateway's /metrics vhost";
|
||
wantedBy = [ "nginx.service" ];
|
||
before = [ "nginx.service" ];
|
||
path = [
|
||
pkgs.coreutils
|
||
pkgs.openssl
|
||
pkgs.systemd
|
||
];
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
UMask = "0077";
|
||
SyslogIdentifier = "swarm-nats-gateway-cert";
|
||
};
|
||
script = ''
|
||
set -eu
|
||
src=${lib.escapeShellArg tlsDir}
|
||
d=${lib.escapeShellArg gatewayTlsDir}
|
||
install -d -m 0755 "$d"
|
||
if [ -s "$src/cert.pem" ] && [ -s "$src/key.pem" ]; then
|
||
cp "$src/cert.pem" "$d/cert.pem.new"
|
||
cp "$src/key.pem" "$d/key.pem.new"
|
||
elif [ -s "$d/cert.pem" ] && [ -s "$d/key.pem" ]; then
|
||
exit 0
|
||
else
|
||
echo "no queue leaf yet — serving a throwaway self-signed certificate on ${cfg.domain} until swarm-bao-nats-tls issues one" >&2
|
||
openssl req -x509 -newkey rsa:2048 -nodes -sha256 -days 30 \
|
||
-subj "/CN=${cfg.domain}" -keyout "$d/key.pem.new" -out "$d/cert.pem.new"
|
||
fi
|
||
chmod 0644 "$d/cert.pem.new"
|
||
chgrp nginx "$d/key.pem.new"
|
||
chmod 0640 "$d/key.pem.new"
|
||
mv -f "$d/key.pem.new" "$d/key.pem"
|
||
mv -f "$d/cert.pem.new" "$d/cert.pem"
|
||
if systemctl is-active --quiet nginx.service; then
|
||
systemctl reload nginx.service
|
||
fi
|
||
'';
|
||
};
|
||
|
||
assertions = [
|
||
{
|
||
# The callout xkey and the responder's OIDC client secret are read
|
||
# from the store under this identity; without either the server or
|
||
# the responder does not start.
|
||
assertion = authStoreActive;
|
||
message = ''
|
||
services.hyperhive.deploy.nats.authBaoClientCertFile and
|
||
authBaoClientKeyFile must both be set.
|
||
|
||
The queue's callout xkey and its responder's OIDC client secret
|
||
are read from the swarm's secret store with this certificate, and
|
||
neither the server nor the responder starts without them. They
|
||
default to the leaf the store mints where
|
||
services.hyperhive.deploy.bao.enable is set on this host;
|
||
elsewhere, issue a leaf whose CN is
|
||
services.hyperhive.deploy.bao.natsAuthCommonName and name it in
|
||
both options.
|
||
'';
|
||
}
|
||
];
|
||
|
||
# Open the client port on the bridge, so agent containers can reach
|
||
# the queue. `privateNetwork = false` below means the server binds in
|
||
# the host netns, which is the precondition this option names — but
|
||
# sharing a netns is not reachability: the bridge interface is
|
||
# default-deny, so without this an agent's connect attempt is dropped
|
||
# by the firewall and looks exactly like every other NATS failure,
|
||
# a timeout.
|
||
#
|
||
# ⚠️ Never the world. What makes it safe to open at all is that the
|
||
# queue is fail-closed: `auth_callout` admits nobody until the responder
|
||
# above answers for them, so an agent that reaches this port still has to
|
||
# present a token authelia vouches for.
|
||
#
|
||
# An agent reaches it by name, which dnsmasq answers with the bridge IP.
|
||
services.hyperhive.network.exposeHostPorts = [ cfg.port ];
|
||
|
||
# The bind source has to exist before the container starts, leaf or not:
|
||
# nixos-container refuses to start over a missing one, which would take
|
||
# the whole queue down rather than only its TLS.
|
||
systemd.tmpfiles.rules = [ "d ${tlsDir} 0755 root root -" ];
|
||
|
||
containers.swarm-nats = {
|
||
autoStart = true;
|
||
ephemeral = false;
|
||
# Journal files on the host, not inside the container: nixpkgs hardcodes
|
||
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
|
||
extraFlags = [ "--link-journal=host" ];
|
||
inherit privateNetwork;
|
||
# The public trust bundle (empty when the gateway is not self-signed)
|
||
# and the queue's own TLS leaf, both read-only.
|
||
bindMounts = caTrust.bindMount // {
|
||
${tlsDir} = {
|
||
hostPath = tlsDir;
|
||
isReadOnly = true;
|
||
};
|
||
};
|
||
config =
|
||
{ ... }:
|
||
{
|
||
imports = [
|
||
../container-modules/swarm-container.nix
|
||
(import ../container-modules/swarm-container-resolver.nix {
|
||
inherit (networkCfg) bridgeIp;
|
||
# The responder introspects authelia BY NAME on every auth
|
||
# request, and a responder that cannot resolve it denies
|
||
# every client — so it must not start before the resolver
|
||
# file exists.
|
||
dnsConsumers = [ "swarm-nats-auth.service" ];
|
||
})
|
||
caBundleModule
|
||
];
|
||
|
||
services.hyperhive.swarmContainer = { inherit privateNetwork; };
|
||
|
||
services.nats = {
|
||
enable = true;
|
||
|
||
# Retention, so a reader can ask "what did this hive last
|
||
# say?" without anyone keeping a second copy. The upstream
|
||
# option also wires `settings.jetstream.store_dir = dataDir`;
|
||
# the container is `ephemeral = false`, so that survives a
|
||
# restart with no bind mount.
|
||
#
|
||
# ⚠️ Losing the store is not a correctness problem here: a
|
||
# reader then sees nothing for every hive, which is the true
|
||
# answer until each one publishes again. It degrades to
|
||
# honesty rather than to a stale "healthy".
|
||
jetstream = true;
|
||
|
||
serverName = "swarm-nats";
|
||
port = cfg.port;
|
||
# Off. The `tls` block below names a leaf that exists only at
|
||
# runtime, and `nats-server -t` loads it, so the build-time check
|
||
# fails on every hive; so do the callout keys, empty in
|
||
# `settings`. Upstream's own description names the case: disable
|
||
# it when the config includes other files. The check moves to
|
||
# server start.
|
||
validateConfig = false;
|
||
|
||
settings =
|
||
calloutBlocks {
|
||
userKey = "";
|
||
issuerKey = "";
|
||
}
|
||
// {
|
||
# The most a single publish may be, in bytes, before the
|
||
# server answers `-ERR 'Maximum Payload Violation'` and
|
||
# closes the connection — not a truncation, a dropped row
|
||
# and a reconnect. This queue carries agent terminal rows
|
||
# published whole rather than split, so upstream's own
|
||
# default is sized to lose one of those rather than merely
|
||
# shorten it. Bounded from above by
|
||
# `max_pending`: nats-server refuses to start once this
|
||
# exceeds it, and widening that ceiling instead costs
|
||
# memory per connection, so this stays comfortably under it.
|
||
max_payload = 8388608;
|
||
|
||
# The monitoring endpoint, which is what the exporter below
|
||
# reads. `//` adds a key `calloutBlocks` does not produce
|
||
# (`accounts`, `authorization`) — checked, because a shallow
|
||
# merge that collided here would drop the auth config while
|
||
# rendering a config the server starts on.
|
||
#
|
||
# ⚠️ Loopback: unauthenticated, and `/connz` lists every
|
||
# connected client.
|
||
http = "127.0.0.1:${toString cfg.monitorPort}";
|
||
|
||
# TLS required on the client port: a server with a `tls`
|
||
# block advertises `tls_required` and refuses a client that
|
||
# does not upgrade. No `allow_non_tls`, which would keep the
|
||
# plaintext path open across the mesh. The 2s timeout is for
|
||
# a handshake that crosses the mesh; upstream's 0.5s is sized
|
||
# for a LAN.
|
||
tls = {
|
||
cert_file = tlsCertPath;
|
||
key_file = tlsKeyCredentialPath;
|
||
timeout = 2;
|
||
};
|
||
};
|
||
};
|
||
|
||
# The key as the `nats` user can read it; see `tlsDir`. Resolved at
|
||
# unit start only, which is why a renewed leaf restarts the server
|
||
# rather than reloading it.
|
||
systemd.services.nats.serviceConfig.LoadCredential = [
|
||
"${tlsKeyCredential}:${tlsKeyPath}"
|
||
];
|
||
|
||
# The translation layer. NATS has no Prometheus format of its
|
||
# own, so this reads the JSON monitoring endpoint above and
|
||
# re-serves it in the format the collector scrapes.
|
||
#
|
||
# Upstream's exporter module rather than a hand-rolled unit, for
|
||
# the same reason `services.nats`'s own `settings` is kept: the
|
||
# reviewed reasoning about flags and hardening lives there.
|
||
services.prometheus.exporters.nats = {
|
||
enable = true;
|
||
listenAddress = "127.0.0.1";
|
||
port = cfg.metricsPort;
|
||
url = "http://127.0.0.1:${toString cfg.monitorPort}";
|
||
|
||
# ⚠️ NOT optional, despite the name. Upstream renders
|
||
# `-addr … -port … ${extraFlags} ${url}` and defaults
|
||
# `extraFlags` to the empty list, but the exporter REFUSES TO
|
||
# START without at least one collector: it exits 1 with
|
||
# "no Collectors specified". Shipped without this the unit logs
|
||
# `Started`, the process is gone milliseconds later, and the
|
||
# scrape is refused — measured, and invisible to evaluation
|
||
# because the empty list renders perfectly.
|
||
#
|
||
# Which three, and why not more: `varz` is the server itself
|
||
# (uptime, memory, slow consumers), `connz` is per-connection so
|
||
# a client that will not stay connected is visible, and `jsz`
|
||
# covers JetStream, which this swarm relies on for the status
|
||
# KV. The remaining collectors describe a clustered deployment
|
||
# this swarm does not have.
|
||
extraFlags = [
|
||
"-varz"
|
||
"-connz"
|
||
"-jsz=all"
|
||
];
|
||
};
|
||
|
||
# A wrapper that includes upstream's rendered settings verbatim
|
||
# plus the runtime fragment; rendering the config ourselves
|
||
# instead would throw away upstream's `settings`, where the
|
||
# reviewed reasoning lives. `renderConfig` writes it at every start,
|
||
# because the includes must be siblings of the fragment (see
|
||
# `runtimeDir`).
|
||
#
|
||
# `mkForce`: upstream defines ExecStart inside an `mkMerge`, so a
|
||
# plain override conflicts rather than wins.
|
||
systemd.services.nats.serviceConfig.ExecStart =
|
||
lib.mkForce "${pkgs.nats-server}/bin/nats-server -c ${runtimeWrapper}";
|
||
systemd.services.nats.serviceConfig.ExecStartPre = "+${renderConfig}";
|
||
|
||
# The auth-callout responder: the half that lets the server
|
||
# above say *yes*. Without it the `auth_callout` block is a
|
||
# door nobody can open, which is the deliberate interim state.
|
||
systemd.services.swarm-nats-auth = {
|
||
description = "swarm queue auth-callout responder";
|
||
after = [ "nats.service" ];
|
||
requires = [ "nats.service" ];
|
||
# `nats.service` too: a server started late, once its keys arrived,
|
||
# brings the responder up with it.
|
||
wantedBy = [
|
||
"multi-user.target"
|
||
"nats.service"
|
||
];
|
||
serviceConfig = {
|
||
ExecStart = lib.concatStringsSep " " [
|
||
"${deployCfg.nats.authPackage}/bin/swarm-nats-auth"
|
||
# By name, like every other client: the leaf carries the name
|
||
# and no IP, so a loopback address fails verification. The
|
||
# container's resolver goes through the bridge, where dnsmasq
|
||
# answers it.
|
||
"--nats-url tls://${cfg.domain}:${toString cfg.port}"
|
||
"--client-id ${lib.escapeShellArg cfg.clientId}"
|
||
# The account admitted clients land in, by NAME: in
|
||
# server-config mode the server resolves `aud` against its
|
||
# own `accounts` block, so this and the block above have to
|
||
# be the same string — which is why both come from one let.
|
||
"--account ${lib.escapeShellArg clientAccount}"
|
||
"--introspection-url ${lib.escapeShellArg introspectionUrl}"
|
||
# Each of these names a principal some OTHER module mints,
|
||
# so each is read out of that module rather than spelled
|
||
# again here — same argument as `--account` above, one
|
||
# level wider. The responder denies a client id it does
|
||
# not recognise, and a NATS denial arrives as a timeout,
|
||
# so a drift here is silent at the point of change and
|
||
# misattributed at the point of failure.
|
||
"--hive-client-prefix ${lib.escapeShellArg autheliaCfg.hiveClientPrefix}"
|
||
"--agent-client-suffix ${lib.escapeShellArg autheliaCfg.agentClientSuffix}"
|
||
"--reader-client ${lib.escapeShellArg controllerCfg.queueClientId}"
|
||
# Hears try-outs and answers each by reply, and nothing else.
|
||
"--review-classifier-client ${lib.escapeShellArg config.services.hyperhive.swarm.review-classifier.clientId}"
|
||
# Hears builds and answers each by reply, and nothing else.
|
||
"--builder-client ${lib.escapeShellArg config.services.hyperhive.swarm.builder.clientId}"
|
||
# What an agent may publish to, keyed on its own name: its
|
||
# terminal rows and its turn-state header
|
||
# (`hive-agent::swarm_agent_state`), two subject families so a
|
||
# subscriber following one agent's terminal is not also handed
|
||
# every header. The hive's shared agent client gets nothing.
|
||
#
|
||
# `$$`, not `$`: systemd substitutes `$NAME` in `ExecStart`
|
||
# whether or not it is quoted, so a single dollar reaches the
|
||
# responder as the empty expansion of an unset `SWARM`.
|
||
"--agent-token-publish-subject ${lib.escapeShellArg "\$\$SWARM.term.{agent}"}"
|
||
"--agent-token-publish-subject ${lib.escapeShellArg "\$\$SWARM.agent-state.{agent}"}"
|
||
# Its own key in the `agent-icons` KV bucket, and nothing else
|
||
# of JetStream. A KV put is a JetStream publish to
|
||
# `$KV.<bucket>.<key>`, acked on the client's inbox, so this
|
||
# one subject is the whole write. The harness publishes to the
|
||
# subject directly instead of opening the bucket, so it needs
|
||
# no `$JS.API.*` subject; swarm-controller creates the bucket.
|
||
"--agent-token-publish-subject ${lib.escapeShellArg "\$\$KV.agent-icons.{agent}"}"
|
||
# Its subagents' terminal rows
|
||
# (`swarm_queue_client::subagent_term`), publish only and no
|
||
# `$JS.API.*` subject: swarm-controller creates the
|
||
# `term-sub-{agent}` stream that keeps them, so its subjects
|
||
# and limits are never the agent's to choose.
|
||
"--agent-token-publish-subject ${lib.escapeShellArg "\$\$SWARM.term.{agent}.sub.>"}"
|
||
# Reading its own key of the `agent-schedules` KV bucket
|
||
# (`swarm_queue_client::agent_schedules`), and nothing else of
|
||
# JetStream: a direct get names the key in its subject, so
|
||
# this one subject is one key. No `STREAM.INFO`, no
|
||
# `CONSUMER.CREATE` (a watch's create subject carries no key,
|
||
# so it would cover every agent's) and no `$KV` write.
|
||
"--agent-token-publish-subject ${lib.escapeShellArg "\$\$JS.API.DIRECT.GET.KV_agent-schedules.\$\$KV.agent-schedules.{agent}"}"
|
||
# Its own key of the `agent-inference` KV bucket
|
||
# (`swarm_queue_client::agent_inference`), on the same terms.
|
||
"--agent-token-publish-subject ${lib.escapeShellArg "\$\$JS.API.DIRECT.GET.KV_agent-inference.\$\$KV.agent-inference.{agent}"}"
|
||
"--store-cert-role ${lib.escapeShellArg authCertRole}"
|
||
];
|
||
# The store identity arrives by `LoadCredential` and is named
|
||
# only as a **path**: `argv` is world-readable via
|
||
# /proc/<pid>/cmdline. Every secret the responder holds beyond
|
||
# it, its own OIDC client secret included, it reads from the
|
||
# store with that identity.
|
||
LoadCredential = lib.mapAttrsToList (id: _: "${id}:${inContainer id}") authStoreFiles;
|
||
DynamicUser = true;
|
||
Restart = "on-failure";
|
||
RestartSec = "5s";
|
||
SyslogIdentifier = "swarm-nats-auth";
|
||
};
|
||
# The store the responder reads the callout keys and agent
|
||
# credentials from.
|
||
environment = {
|
||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||
BAO_CLIENT_CERT = authCredential "bao-client.pem";
|
||
BAO_CLIENT_KEY = authCredential "bao-client-key.pem";
|
||
}
|
||
// lib.optionalAttrs (authStoreFiles ? "bao-ca.pem") {
|
||
BAO_CACERT = authCredential "bao-ca.pem";
|
||
};
|
||
};
|
||
|
||
# The server binary, so an operator with a shell in here can
|
||
# run `nats-server -t` against the generated config. The unit
|
||
# resolves ExecStart through the store path and puts nothing
|
||
# on PATH.
|
||
environment.systemPackages = [ pkgs.nats-server ];
|
||
};
|
||
};
|
||
|
||
# Order the container after the host CA generator, so the bind source
|
||
# exists before nspawn sets the mount up. Without it a late CA fails the
|
||
# container start outright rather than degrading.
|
||
systemd.services."container@${machine}" = caTrust.containerOrdering;
|
||
|
||
# Deliver the responder's store identity into the container before it
|
||
# starts. A copy rather than a `bindMounts` entry: nixos-container refuses
|
||
# to start when a bind source is missing, so one absent file would take
|
||
# down the **whole container including the queue**, not merely the
|
||
# responder.
|
||
systemd.services.swarm-nats-auth-secrets = {
|
||
description = "deliver the swarm queue responder's credentials";
|
||
before = [ "container@swarm-nats.service" ];
|
||
wantedBy = [ "container@swarm-nats.service" ];
|
||
# Where the store's PKI is minted on this host, the responder's leaf is
|
||
# one of its files, and the minting unit runs before every copy. Ordering
|
||
# alone is not enough: the queue's container, which pulls this unit in,
|
||
# starts from `machines.target`, outside the transaction that runs the
|
||
# minting unit from `multi-user.target`. Ignored elsewhere, where the unit
|
||
# does not exist.
|
||
after = [ "swarm-bao-pki.service" ];
|
||
wants = [ "swarm-bao-pki.service" ];
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
SyslogIdentifier = "swarm-nats-auth-secrets";
|
||
};
|
||
path = [ pkgs.coreutils ];
|
||
script = ''
|
||
set -euo pipefail
|
||
install -d -m 0700 ${lib.escapeShellArg secretDir}
|
||
''
|
||
+ lib.concatStrings (
|
||
lib.mapAttrsToList (id: source: ''
|
||
install -m 0400 ${lib.escapeShellArg source} ${lib.escapeShellArg (hostPath id)}
|
||
'') authStoreFiles
|
||
);
|
||
};
|
||
|
||
# The callout's three public keys, from the store into the files
|
||
# `renderConfig` reads at every server start. `ensure-callout-keys` first
|
||
# creates each pair the store holds none of; it needs no queue, so on a
|
||
# first boot the keys exist before the server that requires them, and
|
||
# nothing here waits on the responder, which itself waits on the server.
|
||
#
|
||
# An issuer or user seed file left on this host is what `ensure-callout-keys`
|
||
# stores for that key when the store holds none, so the key keeps its
|
||
# public half. It removes the file once a read of the store returns a seed,
|
||
# in this run and never before: a removal that ran ahead of the import,
|
||
# with the store sealed, would leave the next run to mint a new key.
|
||
#
|
||
# Before the container and wanted by it, the shape `swarm-bao-nats-tls`
|
||
# has: a store that comes up later fails this run, the server refuses to
|
||
# start without the files (or starts with the ones last fetched), and the
|
||
# retry that lands them starts or restarts the server. Under the
|
||
# responder's own leaf, the one principal that may create the keys' paths.
|
||
systemd.services.swarm-bao-nats-callout-xkey = {
|
||
description = "create the swarm queue's callout keys if absent and fetch their public halves for the server";
|
||
# Ordered after the store where it runs here
|
||
# (./glue-bao-readers-store-order.nix); elsewhere the timeout below
|
||
# bounds the attempt.
|
||
before = [ "container@${machine}.service" ];
|
||
wantedBy = [
|
||
"multi-user.target"
|
||
"container@${machine}.service"
|
||
];
|
||
path = [
|
||
deployCfg.nats.authPackage
|
||
baoDeploy.package
|
||
pkgs.coreutils
|
||
pkgs.systemd
|
||
];
|
||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||
serviceConfig = storeRetry.serviceConfig // {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
SyslogIdentifier = "swarm-bao-nats-callout-xkey";
|
||
TimeoutStartSec = 30;
|
||
};
|
||
environment = {
|
||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||
BAO_CLIENT_CERT = deployCfg.nats.authBaoClientCertFile;
|
||
BAO_CLIENT_KEY = deployCfg.nats.authBaoClientKeyFile;
|
||
}
|
||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||
BAO_CACERT = baoDeploy.serverCaFile;
|
||
};
|
||
script = ''
|
||
set -euo pipefail
|
||
|
||
${atomicWriteSecret}
|
||
${refreshConsumer}
|
||
|
||
swarm-nats-auth ensure-callout-keys --store-cert-role ${lib.escapeShellArg authCertRole} \
|
||
--import-issuer-seed-file ${lib.escapeShellArg importIssuerSeed} \
|
||
--import-user-seed-file ${lib.escapeShellArg importUserSeed}
|
||
if [ -d ${lib.escapeShellArg importSeedDir} ]; then
|
||
rmdir --ignore-fail-on-non-empty ${lib.escapeShellArg importSeedDir}
|
||
fi
|
||
|
||
err="$(mktemp)"
|
||
trap 'rm -f "$err"' EXIT
|
||
|
||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||
echo "could not log in to swarm-bao with the queue responder's certificate; leaving the callout key files as they are." >&2
|
||
cat "$err" >&2
|
||
exit 1
|
||
fi
|
||
export BAO_TOKEN
|
||
|
||
install -d -m 0755 ${lib.escapeShellArg hostRuntimeDir}
|
||
''
|
||
+ lib.concatMapStrings (
|
||
k:
|
||
let
|
||
file = "${hostRuntimeDir}/${k.file}";
|
||
in
|
||
''
|
||
if ! key="$(bao kv get -field=value ${lib.escapeShellArg k.storePath} 2>"$err")"; then
|
||
echo "swarm-bao did not return ${k.storePath}." >&2
|
||
cat "$err" >&2
|
||
exit 1
|
||
fi
|
||
if [ -z "$key" ]; then
|
||
echo "swarm-bao returned an empty ${k.storePath}." >&2
|
||
exit 1
|
||
fi
|
||
if secret_differs ${lib.escapeShellArg file} "$key"; then
|
||
atomic_write_secret 0444 "" ${lib.escapeShellArg file} "$key"
|
||
fi
|
||
''
|
||
) calloutPublicKeys
|
||
# `renderConfig` reads the files as the server starts, and only then.
|
||
+ lib.concatMapStrings (k: ''
|
||
refresh_consumer ${lib.escapeShellArg machine} nats.service ${lib.escapeShellArg "${hostRuntimeDir}/${k.file}"}
|
||
'') calloutPublicKeys;
|
||
};
|
||
|
||
# The queue's TLS leaf, issued by the secret store's `pki/roles/swarm-nats`
|
||
# under this host's `swarm-nats` identity. Same shape as ./hive-tls.nix's
|
||
# `swarm-services-cert`, whose comments carry the reasoning for the login,
|
||
# the single `issue` call split with `jq`, and the unseal-sized retry.
|
||
#
|
||
# Before the container, so a normal boot starts the server with its leaf.
|
||
# A store that comes up later fails this run; the retry lands the leaf and
|
||
# restarts the server, which until then refuses to start (TLS required,
|
||
# and its key credential is missing).
|
||
systemd.services.swarm-bao-nats-tls = {
|
||
description = "Issue the swarm queue's TLS leaf from the secret store's PKI";
|
||
wantedBy = [
|
||
"multi-user.target"
|
||
"container@${machine}.service"
|
||
];
|
||
before = [ "container@${machine}.service" ];
|
||
# Both absent where the store runs elsewhere, and ignored there; see
|
||
# `swarm-services-cert`. The ordering after this leaf's policy unit is
|
||
# ./glue-bao-readers-policy-order.nix's, because it holds only where the
|
||
# store is here too.
|
||
after = [
|
||
"container@${baoCfg.machine}.service"
|
||
"swarm-bao-pki.service"
|
||
];
|
||
wants = [ "container@${baoCfg.machine}.service" ];
|
||
path = [
|
||
baoDeploy.package
|
||
pkgs.jq
|
||
pkgs.openssl
|
||
pkgs.coreutils
|
||
pkgs.gnugrep
|
||
pkgs.systemd
|
||
];
|
||
startLimitBurst = 2880;
|
||
startLimitIntervalSec = 90000;
|
||
# No `RemainAfterExit`: the timer below starts this again, and starting
|
||
# an active unit is a no-op.
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
UMask = "0077";
|
||
SyslogIdentifier = "swarm-bao-nats-tls";
|
||
Restart = "on-failure";
|
||
RestartSec = 30;
|
||
};
|
||
environment = {
|
||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||
}
|
||
// lib.optionalAttrs (deployCfg.nats.baoClientCertFile != null) {
|
||
BAO_CLIENT_CERT = deployCfg.nats.baoClientCertFile;
|
||
}
|
||
// lib.optionalAttrs (deployCfg.nats.baoClientKeyFile != null) {
|
||
BAO_CLIENT_KEY = deployCfg.nats.baoClientKeyFile;
|
||
}
|
||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||
BAO_CACERT = baoDeploy.serverCaFile;
|
||
};
|
||
script = ''
|
||
set -euo pipefail
|
||
d=${lib.escapeShellArg tlsDir}
|
||
name=${lib.escapeShellArg cfg.domain}
|
||
install -d -m 0755 "$d"
|
||
|
||
# Missing, past half its window, or naming something other than the
|
||
# configured domain.
|
||
reissue=0
|
||
{ [ -s "$d/cert.pem" ] && [ -s "$d/key.pem" ]; } || reissue=1
|
||
openssl x509 -in "$d/cert.pem" -noout -checkend ${toString leafRenewSeconds} >/dev/null 2>&1 || reissue=1
|
||
openssl x509 -in "$d/cert.pem" -noout -checkhost "$name" 2>/dev/null | grep -q ' does match ' || reissue=1
|
||
|
||
if [ "$reissue" = 0 ]; then
|
||
echo "queue leaf valid for $name — leaving it alone"
|
||
exit 0
|
||
fi
|
||
|
||
${
|
||
if deployCfg.nats.baoClientCertFile == null || deployCfg.nats.baoClientKeyFile == null then
|
||
''
|
||
echo "no bao client certificate configured for the queue, so its TLS leaf" >&2
|
||
echo "cannot be requested from the store." >&2
|
||
echo "Set services.hyperhive.deploy.nats.baoClient{Cert,Key}File" >&2
|
||
echo "to a leaf the store's CA signed with CN=${baoDeploy.natsCommonName}." >&2
|
||
echo "They default to the leaf the store mints only where" >&2
|
||
echo "services.hyperhive.deploy.bao.enable is set on this host." >&2
|
||
exit 1''
|
||
else
|
||
""
|
||
}
|
||
|
||
err="$(mktemp)"
|
||
trap 'rm -f "$err"' EXIT
|
||
|
||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||
echo "could not log in to the swarm secret store with the queue's certificate." >&2
|
||
cat "$err" >&2
|
||
exit 1
|
||
fi
|
||
export BAO_TOKEN
|
||
|
||
echo "requesting the queue's TLS leaf for $name from the store"
|
||
resp="$(mktemp "$d/issue.json.XXXXXX")"
|
||
trap 'rm -f "$err" "$resp"' EXIT
|
||
|
||
if ! bao write -format=json \
|
||
${lib.escapeShellArg "${baoDeploy.servicesPkiMountPath}/issue/${baoDeploy.natsPkiRoleName}"} \
|
||
common_name="$name" > "$resp" 2>"$err"; then
|
||
echo "the store refused to issue the queue's certificate." >&2
|
||
cat "$err" >&2
|
||
exit 1
|
||
fi
|
||
|
||
jq -r '.data.private_key // empty' < "$resp" > "$d/key.pem.new"
|
||
jq -r '.data.certificate // empty' < "$resp" > "$d/cert.pem.new"
|
||
for f in "$d/key.pem.new" "$d/cert.pem.new"; do
|
||
if [ ! -s "$f" ]; then
|
||
echo "the store's response was missing a field: $f is empty" >&2
|
||
exit 1
|
||
fi
|
||
done
|
||
chmod 0600 "$d/key.pem.new"
|
||
chmod 0644 "$d/cert.pem.new"
|
||
mv -f "$d/key.pem.new" "$d/key.pem"
|
||
mv -f "$d/cert.pem.new" "$d/cert.pem"
|
||
${lib.optionalString gatewayServesQueueLeaf ''
|
||
|
||
# The gateway's `/metrics` vhost serves a copy of this leaf.
|
||
systemctl start swarm-nats-gateway-cert.service''}
|
||
|
||
# Renewal, or the late-store retry. On a normal boot the container is
|
||
# not up yet and starts the server with this leaf itself. A restart,
|
||
# not a reload: the key arrives by `LoadCredential`, which systemd
|
||
# resolves at start only. `reset-failed` because a server that found
|
||
# no key may have hit its start limit.
|
||
if systemctl is-active --quiet ${lib.escapeShellArg "container@${machine}.service"}; then
|
||
echo "queue leaf rotated — restarting the server in ${machine}"
|
||
systemctl --machine=${lib.escapeShellArg machine} reset-failed nats.service
|
||
systemctl --machine=${lib.escapeShellArg machine} restart nats.service
|
||
fi
|
||
'';
|
||
};
|
||
|
||
systemd.timers.swarm-bao-nats-tls = {
|
||
description = "Daily renewal check for the swarm queue's TLS leaf";
|
||
wantedBy = [ "timers.target" ];
|
||
timerConfig = {
|
||
OnCalendar = "daily";
|
||
Persistent = true;
|
||
};
|
||
};
|
||
};
|
||
}
|