d5de1e4943
A new swarm service, `swarm-claude-login`, in its own container on the host
that sets `deploy.claude-login.enable` (defaulted on by `allSwarmServices`
where the host has the minter's store leaf, like the review classifier).
- swarm-ui: an agent card showing "login needed" gets a "log in" button.
Its dialog starts the login, shows the URL claude printed, and takes the
pasted code.
- swarm-controller: `POST/DELETE /api/agents/{name}/claude-login` and
`POST /api/agents/{name}/claude-login/code`, behind the swarm UI's
authelia-gated `/api/`, relay the steps to the minter as requests on
`$SWARM.claude-login.request`.
- swarm-claude-login: runs `claude auth login` over piped stdio with HOME
and CLAUDE_CONFIG_DIR in a per-login scratch dir, scrapes the first
https URL, writes the code to stdin, and stores the `.credentials.json`
at `swarm/agents/<agent>/claude-credentials` with a KV v2 write, a new
version over any earlier one. The scratch dir goes on every exit path; a
login waits 15 minutes for its code.
- hive-agent: while parked in `wait_for_login`, reads that leaf every 30 s
under the agent's own cert and installs it only when its KV version is
newer than the one recorded in `<harness>/claude-credentials-version`.
An equal or older version is never put back, because claude rotates the
refresh token and the stored copy is stale after the first refresh.
- swarm-secret-client: `claude_login` paths and `SecretStore::read_versioned`.
- Credentials: a `claude-login` store leaf minted by glue-bao-tls, a
`swarm-claude-login` policy (create/update on
`swarm/agents/+/claude-credentials`, read on its own queue client
secret), an authelia machine client, and a `--claude-login-client`
grant in the queue's auth-callout responder. The unit runs as a static
system user.
The hive's own web-UI login is unchanged.
Refs #5268
1158 lines
53 KiB
Nix
1158 lines
53 KiB
Nix
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
cfg = config.services.hyperhive.swarm.nats;
|
||
autheliaCfg = config.services.hyperhive.swarm.authelia;
|
||
deployCfg = config.services.hyperhive.deploy;
|
||
autheliaUrl = autheliaCfg.url;
|
||
networkCfg = config.services.hyperhive.network;
|
||
# Read even when the controller runs on a different host: what is needed
|
||
# is the client id that module *declares*, which is the same string
|
||
# everywhere, not whether the daemon happens to be enabled here.
|
||
controllerCfg = config.services.hyperhive.swarm.controller;
|
||
|
||
# The account the callout responder authenticates as, and the account
|
||
# authorized clients are placed in. Two accounts rather than one: an
|
||
# account is NATS' isolation boundary, so a responder that shares an
|
||
# account with its clients can be published to by the things it
|
||
# authorizes.
|
||
calloutAccount = "AUTH";
|
||
clientAccount = "APP";
|
||
|
||
machine = "swarm-nats";
|
||
|
||
tlsCfg = config.services.hyperhive.deploy.hive-controller.tls;
|
||
gatewayCfg = config.services.hyperhive.gateway;
|
||
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
||
# The responder introspects authelia over https BY NAME. Its HTTP client is
|
||
# reqwest/rustls, and `rustls-platform-verifier` resolves roots through
|
||
# `rustls-native-certs`, which reads `SSL_CERT_FILE` — so the same assembled
|
||
# bundle the Go containers use applies here. Without it the handshake fails
|
||
# `UnknownIssuer`, introspection fails, and the responder denies *every*
|
||
# client: one missing trust anchor surfacing as `authorization violation` at
|
||
# every would-be queue user.
|
||
caBundleModule = caTrust.trustBundle {
|
||
inherit pkgs;
|
||
name = machine;
|
||
consumers = [ "swarm-nats-auth" ];
|
||
};
|
||
# Where the responder's credentials live *inside* the container, and the
|
||
# host path that resolves to. Two names for one location, because the
|
||
# host is the only place both filesystems are addressable.
|
||
secretDirInContainer = "/var/lib/swarm-nats-auth";
|
||
inContainer = name: "${secretDirInContainer}/${name}";
|
||
secretDir = "/var/lib/nixos-containers/${machine}${secretDirInContainer}";
|
||
hostPath = name: "${secretDir}/${name}";
|
||
|
||
introspectionUrl = "${toString autheliaUrl}/api/oidc/introspection";
|
||
|
||
# Host seed files `ensure-callout-keys` stores when the secret store holds
|
||
# no seed for that key, and removes once it holds one. Never written here.
|
||
importSeedDir = "/var/lib/swarm-nats-callout";
|
||
importUserSeed = "${importSeedDir}/callout-user.seed";
|
||
importIssuerSeed = "${importSeedDir}/issuer.seed";
|
||
|
||
# The runtime config directory: wrapper, settings symlink and fragment.
|
||
# World-readable is correct (everything in it is public) and it is not in
|
||
# the 0700 responder secret dir, which the `nats` user cannot traverse.
|
||
#
|
||
# ⚠️ ONE DIRECTORY IS FORCED, NOT TIDINESS. NATS resolves an `include` with
|
||
# `filepath.Join(configDir, path)`, which strips a leading slash, so an
|
||
# absolute include silently becomes relative and is never found. The
|
||
# includes must therefore be bare filenames, i.e. siblings. Invisible to
|
||
# eval: the wrapper renders perfectly and the server refuses to start.
|
||
runtimeDir = "/var/lib/nats-callout";
|
||
runtimeWrapper = "${runtimeDir}/nats.conf";
|
||
hostRuntimeDir = "/var/lib/nixos-containers/${machine}${runtimeDir}";
|
||
|
||
natsFormat = pkgs.formats.json { };
|
||
|
||
# Upstream's rendered settings, re-rendered with the same generator on the
|
||
# same value — the artifact `services.nats` would have used, not a
|
||
# transcription. ⚠️ `renderConfig` below is the only thing naming it:
|
||
# `ExecStart` names a runtime path, so that reference is what keeps the store
|
||
# file in the closure and stops it being garbage-collected out from under a
|
||
# running server.
|
||
renderedSettings = natsFormat.generate "nats.conf" (
|
||
config.containers.${machine}.config.services.nats.settings
|
||
);
|
||
|
||
# Defined once, rendered twice: into `settings` with no keys, and into the
|
||
# runtime fragment with placeholders for the three public keys, filled in
|
||
# at server start.
|
||
#
|
||
# 🪤 Do not inline these and hand-write the fragment instead. Both files are
|
||
# loaded and the fragment is the LATER definition, so it wins — a
|
||
# hand-written copy would make a future edit to `settings` silently
|
||
# ineffective.
|
||
calloutBlocks =
|
||
{
|
||
userKey,
|
||
issuerKey,
|
||
xkey ? null,
|
||
}:
|
||
{
|
||
# Two accounts, and the callout user lives in neither of the accounts
|
||
# it authorizes into.
|
||
accounts = {
|
||
# ⚠️ The nkey is not decoration, and without it there is a hole: a
|
||
# `users` entry carrying only a `user` name has no credential, and
|
||
# `CONNECT {"user":"auth"}` is then accepted with no password at
|
||
# all. Since the name is a literal in this public module, that made
|
||
# the callout-exempt identity walk-in-able from every container on
|
||
# the shared netns — the same class of hole this module exists to
|
||
# close. An nkey and NOTHING else, both halves
|
||
# measured against a running server rather than reasoned about:
|
||
#
|
||
# { user = "auth"; } → `CONNECT {"user":"auth"}`
|
||
# is accepted with no
|
||
# credential at all
|
||
# { user = "auth"; nkey = "U…"; } → refuses to START:
|
||
# "Nkey users do not take
|
||
# usernames or passwords"
|
||
# { nkey = "U…"; } → what this is
|
||
#
|
||
# A malformed key is fail-closed too: the server exits with
|
||
# "Not a valid public nkey for a user" rather than starting with a
|
||
# hole. So the only way to get a live server here is a real key
|
||
# whose seed nobody but the responder holds.
|
||
#
|
||
# 🔒 That property is what makes `settings` safe: both nkeys render
|
||
# as "" there, so any field the fragment fails to override keeps a
|
||
# value the server refuses to start on. An incomplete merge cannot leave
|
||
# a walk-in-able server.
|
||
${calloutAccount}.users = [ { nkey = userKey; } ];
|
||
# ⚠️ `services.nats.jetstream = true` gives the SERVER JetStream;
|
||
# an account gets it only from its own grant. Measured against a
|
||
# running 2.14.1 with this exact two-account shape, because the
|
||
# failure is invisible to any config-rendering check:
|
||
#
|
||
# global jetstream only → `nats kv add` from this account fails
|
||
# `code=503 err_code=10039 jetstream not enabled for account`,
|
||
# while the server starts cleanly and logs "Starting JetStream"
|
||
# + this line → the same command succeeds
|
||
#
|
||
# The grant is per-account by design, and that is worth keeping:
|
||
# the callout account above deliberately does NOT get it. The
|
||
# responder mints credentials; it has no business holding stream
|
||
# state.
|
||
#
|
||
# ⚠️ Also why the fragment renders the COMPLETE accounts block: if a
|
||
# later definition replaced rather than merged, a partial one would
|
||
# drop this grant and every KV op would fail on a healthy server.
|
||
${clientAccount} = {
|
||
jetstream = "enabled";
|
||
};
|
||
};
|
||
|
||
authorization = {
|
||
timeout = "2s";
|
||
# 🔒 THIS BLOCK IS THE FAIL-CLOSED STATE, and it is the measured
|
||
# one rather than the obvious one.
|
||
#
|
||
# Measured on the pinned nats-server 2.14.1: both
|
||
# `authorization { }` and `authorization { users: [] }` accept an
|
||
# anonymous client and answer PONG — they read like "authorize
|
||
# nobody" and are wide open. An auth_callout block sets
|
||
# `auth_required` and refuses every client whose credential no
|
||
# responder has approved, so a config whose responder does not
|
||
# exist yet denies everyone.
|
||
#
|
||
# `nats-server -t` calls all three valid; it parses, it does not
|
||
# authenticate. Only running them tells the difference.
|
||
#
|
||
# ⇒ this is both the safe interim state and the final shape.
|
||
# Nothing here has to be swapped out when the responder lands
|
||
# beside it — it only starts being able to say yes.
|
||
auth_callout = {
|
||
issuer = issuerKey;
|
||
auth_users = [ userKey ];
|
||
account = calloutAccount;
|
||
}
|
||
# The curve key the server seals each request to, and opens the
|
||
# reply with. A key that does not parse stops the server starting.
|
||
// lib.optionalAttrs (xkey != null) { inherit xkey; };
|
||
};
|
||
};
|
||
|
||
# The callout's three public keys: where the store keeps each, the file
|
||
# `swarm-bao-nats-callout-xkey` leaves it in, the placeholder it fills in
|
||
# `calloutTemplate`, and the shape `renderConfig` checks it has. The store
|
||
# paths are `swarm-nats-auth`'s `xkey::PUBLIC_PATH` and the `public_path` of
|
||
# its `nkey::ISSUER` and `nkey::USER`, readable by the `swarm-nats-auth` role
|
||
# alone; each seed beside them is in ./swarm-bao.nix.
|
||
calloutPublicKeys = [
|
||
{
|
||
storePath = "secret/swarm/public/swarm-nats/callout-xkey";
|
||
file = "callout-xkey.pub";
|
||
placeholder = "@XKEY_PUBKEY@";
|
||
var = "xkey";
|
||
pattern = "X[A-Z2-7]{55}";
|
||
kind = "public xkey";
|
||
}
|
||
{
|
||
storePath = "secret/swarm/public/swarm-nats/callout-issuer";
|
||
file = "issuer.pub";
|
||
placeholder = "@ISSUER_PUBKEY@";
|
||
var = "issuer_pub";
|
||
pattern = "A[A-Z2-7]{55}";
|
||
kind = "public account nkey";
|
||
}
|
||
{
|
||
storePath = "secret/swarm/public/swarm-nats/callout-user";
|
||
file = "callout-user.pub";
|
||
placeholder = "@USER_PUBKEY@";
|
||
var = "user_pub";
|
||
pattern = "U[A-Z2-7]{55}";
|
||
kind = "public user nkey";
|
||
}
|
||
];
|
||
|
||
# The fragment as nix renders it, with placeholders where the runtime
|
||
# values go. Rendered by the same JSON generator upstream uses, so the
|
||
# fragment is generated rather than transcribed — NATS' config parser
|
||
# accepts JSON, and an `include` of it merges (measured).
|
||
calloutTemplate = natsFormat.generate "swarm-nats-callout-template.conf" (calloutBlocks {
|
||
userKey = "@USER_PUBKEY@";
|
||
issuerKey = "@ISSUER_PUBKEY@";
|
||
xkey = "@XKEY_PUBKEY@";
|
||
});
|
||
|
||
# Each placeholder in `calloutTemplate`, as a `sed` expression over the
|
||
# shell variable `renderConfig` reads its value into.
|
||
calloutSubstitutions = map (k: ''-e "s|${k.placeholder}|${"$"}${k.var}|g"'') calloutPublicKeys;
|
||
|
||
# Renders the server's config as it starts, so the settings it includes are
|
||
# always the running system's. Exits non-zero, and so stops the server
|
||
# starting, while any of the three public keys has not been fetched: the
|
||
# server never runs without them. `+` because the server's own sandbox
|
||
# cannot write `runtimeDir`.
|
||
renderConfig = pkgs.writeShellScript "swarm-nats-render-config" ''
|
||
set -euo pipefail
|
||
PATH=${
|
||
lib.makeBinPath [
|
||
pkgs.coreutils
|
||
pkgs.gnugrep
|
||
pkgs.gnused
|
||
]
|
||
}
|
||
d=${lib.escapeShellArg runtimeDir}
|
||
|
||
${lib.concatMapStrings (k: ''
|
||
if [ ! -s "$d/${k.file}" ]; then
|
||
echo "${runtimeDir}/${k.file} is missing: swarm-bao-nats-callout-xkey on the host fetches it from the secret store, and the queue does not start without it" >&2
|
||
exit 1
|
||
fi
|
||
${k.var}="$(cat "$d/${k.file}")"
|
||
# The key type's letter and 55 base32 characters; checked here because
|
||
# it is about to become part of a sed expression.
|
||
if ! printf '%s' "${"$"}${k.var}" | grep -Eqx '${k.pattern}'; then
|
||
echo "${runtimeDir}/${k.file} does not hold a ${k.kind}" >&2
|
||
exit 1
|
||
fi
|
||
'') calloutPublicKeys}
|
||
sed ${lib.concatStringsSep " " calloutSubstitutions} ${calloutTemplate} > "$d/callout.conf.tmp"
|
||
# Mode BEFORE the rename: a rename publishes whatever the file already
|
||
# is. World-readable, because the server reads it as `nats` and
|
||
# everything in it is public.
|
||
chmod 0444 "$d/callout.conf.tmp"
|
||
mv "$d/callout.conf.tmp" "$d/callout.conf"
|
||
|
||
# Upstream's rendered settings, as a sibling the wrapper can name without
|
||
# a leading slash.
|
||
ln -sfn ${renderedSettings} "$d/settings.conf"
|
||
|
||
# The wrapper. Bare filenames — see `runtimeDir`. printf rather than a
|
||
# heredoc, whose terminator would depend on nix's indentation stripping.
|
||
printf 'include "settings.conf"\ninclude "callout.conf"\n' > "$d/nats.conf.tmp"
|
||
chmod 0444 "$d/nats.conf.tmp"
|
||
mv "$d/nats.conf.tmp" "$d/nats.conf"
|
||
'';
|
||
|
||
baoCfg = config.services.hyperhive.swarm.bao;
|
||
baoDeploy = deployCfg.bao;
|
||
|
||
# The queue's TLS leaf, bound read-only at the same path on both sides, the
|
||
# shape ./swarm-bao.nix's `tlsDir` uses. The key reaches the server through
|
||
# `LoadCredential`, as openbao's does: it is root-owned 0600 here, and the
|
||
# server runs as `nats`.
|
||
tlsDir = "/var/lib/swarm-nats-tls";
|
||
tlsCertPath = "${tlsDir}/cert.pem";
|
||
tlsKeyPath = "${tlsDir}/key.pem";
|
||
tlsKeyCredential = "tls-key";
|
||
tlsKeyCredentialPath = "/run/credentials/nats.service/${tlsKeyCredential}";
|
||
|
||
# The gateway's copy of that leaf, for the `/metrics` vhost below: the key
|
||
# here is 0640 root:nginx, so nginx reads it while the queue's own stays
|
||
# 0600 root. Self-signed mode only, the mode `gateway.lib.tlsFor` hands the
|
||
# swarm-services leaf to; with ACME or an operator cert that issuer covers
|
||
# this name as it covers every other.
|
||
gatewayServesQueueLeaf = gatewayCfg.enable && gatewayCfg.useSelfSigned;
|
||
gatewayTlsDir = "/var/lib/swarm-nats-gateway-tls";
|
||
|
||
# The responder's own store identity, for the callout xkey, agent queue
|
||
# credentials and its OIDC client secret. Required: an assertion below refuses
|
||
# a queue without it.
|
||
authStoreActive =
|
||
deployCfg.nats.authBaoClientCertFile != null && deployCfg.nats.authBaoClientKeyFile != null;
|
||
# The role ./swarm-bao.nix writes on the store's `cert` mount, by the same name.
|
||
authCertRole = "swarm-nats-auth";
|
||
# Store identity files the copy unit delivers, as credential id → host source.
|
||
authStoreFiles = {
|
||
"bao-client.pem" = deployCfg.nats.authBaoClientCertFile;
|
||
"bao-client-key.pem" = deployCfg.nats.authBaoClientKeyFile;
|
||
}
|
||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||
"bao-ca.pem" = baoDeploy.serverCaFile;
|
||
};
|
||
authCredential = id: "/run/credentials/swarm-nats-auth.service/${id}";
|
||
|
||
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
|
||
refreshConsumer = import ./lib/refresh-consumer.nix { };
|
||
storeRetry = import ./lib/store-retry.nix { };
|
||
|
||
# Re-issue once the leaf is past half of the 720h `pki/roles/swarm-nats`
|
||
# grants it (./swarm-bao.nix), so the daily timer below has two weeks of
|
||
# retries before it lapses.
|
||
leafRenewSeconds = 360 * 3600;
|
||
|
||
# Shared host netns, like every sibling swarm container.
|
||
#
|
||
# ⚠️ Which is exactly why the server below must refuse everyone
|
||
# until the callout responder exists: the queue is on the host's
|
||
# own loopback, in reach of every process there and every sibling
|
||
# on this netns, and each remote hive in a multi-host swarm dials
|
||
# it directly. An unauthenticated interim state would be a hole
|
||
# rather than a rough edge.
|
||
#
|
||
# Not agent containers, though: they have a netns of their own and
|
||
# the bridge firewall does not open this port.
|
||
privateNetwork = false;
|
||
in
|
||
{
|
||
# The swarm's message queue: one NATS server, reached by every hive at
|
||
# `tls://<swarm.nats.domain>:<port>`.
|
||
#
|
||
# ⚠️ The gateway does NOT front the queue. NATS speaks its own TCP protocol
|
||
# rather than HTTP, so the server terminates TLS itself on `port`. The
|
||
# name is the half of the sibling pattern that applies, the way it is for
|
||
# bao: `gateway.localNames` on this host, the operator's DNS everywhere
|
||
# else. The one gateway vhost on that name serves the exporter's
|
||
# `/metrics` to the swarm collector and nothing else.
|
||
|
||
# What the queue IS to every hive, the ports it answers on and the client id
|
||
# it is registered under, is `swarm.nats` in ./swarm-nats-service.nix. What
|
||
# the host running it decides is below — which responder build it runs and
|
||
# the store identities its units present. `enable` already lives in
|
||
# ./deploy.nix, which also carries the renames. The callout's keys are no
|
||
# option at all: they live in the secret store (docs/swarm/secrets.md).
|
||
options.services.hyperhive.deploy.nats = {
|
||
authPackage = lib.mkOption {
|
||
type = lib.types.package;
|
||
defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-nats-auth";
|
||
description = ''
|
||
The auth-callout responder package.
|
||
|
||
⚠️ Named `authPackage`, not `package`, on purpose: this module
|
||
deliberately has **no** `package` option for the NATS server
|
||
itself, because upstream's `services.nats` resolves
|
||
`pkgs.nats-server` on its own — see the note above
|
||
`options.services.hyperhive.swarm.nats`. A bare `package` here
|
||
would read as "the NATS package" and mean something else
|
||
entirely.
|
||
'';
|
||
};
|
||
|
||
baoClientCertFile = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/var/lib/swarm-bao-pki/nats.pem";
|
||
description = ''
|
||
Client certificate `swarm-bao-nats-tls` presents to the swarm's
|
||
secret store when it asks for the queue's TLS leaf. Its subject must
|
||
be {option}`services.hyperhive.deploy.bao.natsCommonName`: cert auth
|
||
matches on the CN, and that role's policy may issue from
|
||
`pki/issue/swarm-nats` and nothing else.
|
||
|
||
No default. ./glue-nats-bao-identity.nix points it at the leaf
|
||
./glue-bao-tls.nix mints on a host that runs the store
|
||
({option}`services.hyperhive.deploy.bao.enable`). On a queue host
|
||
without the store, it is a file the operator copies from the store's
|
||
host. Never derived from
|
||
{option}`services.hyperhive.deploy.bao.clientCertFile`, which is the
|
||
hive's own reader.
|
||
|
||
A path, never a value.
|
||
'';
|
||
};
|
||
|
||
baoClientKeyFile = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/var/lib/swarm-bao-pki/nats-key.pem";
|
||
description = ''
|
||
Private key for {option}`services.hyperhive.deploy.nats.baoClientCertFile`.
|
||
A path, never a value.
|
||
'';
|
||
};
|
||
|
||
authBaoClientCertFile = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/var/lib/swarm-bao-pki/nats-auth.pem";
|
||
description = ''
|
||
Client certificate the auth-callout responder presents to the swarm's
|
||
secret store to read the callout keys, its own OIDC client secret and
|
||
agent queue credentials, and that `swarm-bao-nats-callout-xkey`
|
||
presents to read the callout keys' public halves. Its subject must be
|
||
{option}`services.hyperhive.deploy.bao.natsAuthCommonName`.
|
||
|
||
Required when `deploy.nats.enable` is set; unset fails at eval.
|
||
./glue-nats-auth-bao-identity.nix points it at the leaf
|
||
./glue-bao-tls.nix mints on a host that runs the store
|
||
({option}`services.hyperhive.deploy.bao.enable`); on any other queue
|
||
host it is a file the operator places. Never derived from
|
||
{option}`services.hyperhive.deploy.bao.clientCertFile`, which is the
|
||
hive's own reader. Set to a file that does
|
||
not exist, `swarm-nats-auth-secrets` and `swarm-bao-nats-callout-xkey`
|
||
fail, and the queue's server does not start without the xkey.
|
||
|
||
A path, never a value.
|
||
'';
|
||
};
|
||
|
||
authBaoClientKeyFile = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/var/lib/swarm-bao-pki/nats-auth-key.pem";
|
||
description = ''
|
||
Private key for {option}`services.hyperhive.deploy.nats.authBaoClientCertFile`.
|
||
A path, never a value.
|
||
'';
|
||
};
|
||
};
|
||
|
||
config = lib.mkIf deployCfg.nats.enable {
|
||
# The auth responder introspects authelia by name, and the gateway serves
|
||
# the exporter's `/metrics` below.
|
||
services.hyperhive.gateway.enable = lib.mkDefault true;
|
||
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
||
|
||
# The name every client dials, answered with the bridge IP on this host.
|
||
# Other hives resolve it through the operator's DNS, as they do bao's.
|
||
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
||
|
||
# The exporter's metrics, for the swarm collector wherever it runs, and
|
||
# nothing else on this name: the monitoring endpoint (`/varz`, `/connz`)
|
||
# is never proxied, and every other path is 403.
|
||
#
|
||
# ⚠️ Not `gateway.lib.tlsFor`, and the name is not in
|
||
# `swarm.serviceDomains`. In self-signed mode this vhost serves the
|
||
# queue's own leaf (issued for exactly this name, `server_flag` set), so
|
||
# a host whose only swarm service is the queue never fronts a services
|
||
# name and its gateway does not wait for the swarm-services leaf.
|
||
#
|
||
# `forceSSL` for the reason ./swarm-victorialogs.nix gives: authelia
|
||
# answers a plain-http auth subrequest with 400.
|
||
services.nginx.virtualHosts."${cfg.domain}" =
|
||
(
|
||
if gatewayServesQueueLeaf then
|
||
{
|
||
sslCertificate = "${gatewayTlsDir}/cert.pem";
|
||
sslCertificateKey = "${gatewayTlsDir}/key.pem";
|
||
}
|
||
else
|
||
builtins.removeAttrs (gatewayCfg.lib.tlsFor cfg.domain) [ "addSSL" ]
|
||
)
|
||
// {
|
||
forceSSL = true;
|
||
listen = gatewayCfg.lib.listen;
|
||
extraConfig = gatewayCfg.lib.securityHeaders;
|
||
locations = {
|
||
"/".return = "403";
|
||
|
||
# The collector requests exactly this URL, declared as
|
||
# `publishedScrapeTargets.nats` in ./swarm-otel-service.nix; the
|
||
# two must agree, since that string is also the token's audience.
|
||
# Authelia's rules for it admit the collector's client and deny
|
||
# every other subject (`metricsRules` in ./swarm-authelia.nix).
|
||
"= /metrics" = {
|
||
proxyPass = "http://127.0.0.1:${toString cfg.metricsPort}/metrics";
|
||
extraConfig = ''
|
||
auth_request /__nats_metrics_authz;
|
||
'';
|
||
};
|
||
|
||
# The subrequest, same shape as the forge's `= /__forge_metrics_authz`
|
||
# (./hive-forge/default.nix): no `error_page 401 =302`, so a scraper
|
||
# gets the 401 rather than a login page parsed as metrics.
|
||
"= /__nats_metrics_authz" = {
|
||
proxyPass = "https://${autheliaCfg.domain}/api/authz/auth-request";
|
||
# nixpkgs appends its OWN `Host $host` after extraConfig, which
|
||
# would override verifiedProxyTo's — see hive-gateway/vhost-lib.nix.
|
||
recommendedProxySettings = false;
|
||
extraConfig = ''
|
||
internal;
|
||
${gatewayCfg.lib.verifiedProxyTo autheliaCfg.domain}
|
||
proxy_pass_request_body off;
|
||
proxy_set_header Content-Length "";
|
||
proxy_set_header X-Original-Method $request_method;
|
||
proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
|
||
proxy_set_header X-Forwarded-Proto $scheme;
|
||
proxy_set_header X-Forwarded-Host $http_host;
|
||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||
'';
|
||
};
|
||
};
|
||
};
|
||
|
||
# Puts a pair at the paths the vhost above names, before nginx starts:
|
||
# nginx refuses a config naming a missing cert file, and that takes the
|
||
# whole gateway down rather than this one name. The queue's leaf when it
|
||
# exists, else a throwaway self-signed pair until `swarm-bao-nats-tls`
|
||
# issues the leaf and starts this again. Never fails on a missing leaf.
|
||
systemd.services.swarm-nats-gateway-cert = lib.mkIf gatewayServesQueueLeaf {
|
||
description = "Copy the swarm queue's TLS leaf for the gateway's /metrics vhost";
|
||
wantedBy = [ "nginx.service" ];
|
||
before = [ "nginx.service" ];
|
||
path = [
|
||
pkgs.coreutils
|
||
pkgs.openssl
|
||
pkgs.systemd
|
||
];
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
UMask = "0077";
|
||
SyslogIdentifier = "swarm-nats-gateway-cert";
|
||
};
|
||
script = ''
|
||
set -eu
|
||
src=${lib.escapeShellArg tlsDir}
|
||
d=${lib.escapeShellArg gatewayTlsDir}
|
||
install -d -m 0755 "$d"
|
||
if [ -s "$src/cert.pem" ] && [ -s "$src/key.pem" ]; then
|
||
cp "$src/cert.pem" "$d/cert.pem.new"
|
||
cp "$src/key.pem" "$d/key.pem.new"
|
||
elif [ -s "$d/cert.pem" ] && [ -s "$d/key.pem" ]; then
|
||
exit 0
|
||
else
|
||
echo "no queue leaf yet — serving a throwaway self-signed certificate on ${cfg.domain} until swarm-bao-nats-tls issues one" >&2
|
||
openssl req -x509 -newkey rsa:2048 -nodes -sha256 -days 30 \
|
||
-subj "/CN=${cfg.domain}" -keyout "$d/key.pem.new" -out "$d/cert.pem.new"
|
||
fi
|
||
chmod 0644 "$d/cert.pem.new"
|
||
chgrp nginx "$d/key.pem.new"
|
||
chmod 0640 "$d/key.pem.new"
|
||
mv -f "$d/key.pem.new" "$d/key.pem"
|
||
mv -f "$d/cert.pem.new" "$d/cert.pem"
|
||
if systemctl is-active --quiet nginx.service; then
|
||
systemctl reload nginx.service
|
||
fi
|
||
'';
|
||
};
|
||
|
||
assertions = [
|
||
{
|
||
# The callout xkey and the responder's OIDC client secret are read
|
||
# from the store under this identity; without either the server or
|
||
# the responder does not start.
|
||
assertion = authStoreActive;
|
||
message = ''
|
||
services.hyperhive.deploy.nats.authBaoClientCertFile and
|
||
authBaoClientKeyFile must both be set.
|
||
|
||
The queue's callout xkey and its responder's OIDC client secret
|
||
are read from the swarm's secret store with this certificate, and
|
||
neither the server nor the responder starts without them. They
|
||
default to the leaf the store mints where
|
||
services.hyperhive.deploy.bao.enable is set on this host;
|
||
elsewhere, issue a leaf whose CN is
|
||
services.hyperhive.deploy.bao.natsAuthCommonName and name it in
|
||
both options.
|
||
'';
|
||
}
|
||
];
|
||
|
||
# Open the client port on the bridge, so agent containers can reach
|
||
# the queue. `privateNetwork = false` below means the server binds in
|
||
# the host netns, which is the precondition this option names — but
|
||
# sharing a netns is not reachability: the bridge interface is
|
||
# default-deny, so without this an agent's connect attempt is dropped
|
||
# by the firewall and looks exactly like every other NATS failure,
|
||
# a timeout.
|
||
#
|
||
# ⚠️ Never the world. What makes it safe to open at all is that the
|
||
# queue is fail-closed: `auth_callout` admits nobody until the responder
|
||
# above answers for them, so an agent that reaches this port still has to
|
||
# present a token authelia vouches for.
|
||
#
|
||
# An agent reaches it by name, which dnsmasq answers with the bridge IP.
|
||
services.hyperhive.network.exposeHostPorts = [ cfg.port ];
|
||
|
||
# The bind source has to exist before the container starts, leaf or not:
|
||
# nixos-container refuses to start over a missing one, which would take
|
||
# the whole queue down rather than only its TLS.
|
||
systemd.tmpfiles.rules = [ "d ${tlsDir} 0755 root root -" ];
|
||
|
||
containers.swarm-nats = {
|
||
autoStart = true;
|
||
ephemeral = false;
|
||
# Journal files on the host, not inside the container: nixpkgs hardcodes
|
||
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
|
||
extraFlags = [ "--link-journal=host" ];
|
||
inherit privateNetwork;
|
||
# The public trust bundle (empty when the gateway is not self-signed)
|
||
# and the queue's own TLS leaf, both read-only.
|
||
bindMounts = caTrust.bindMount // {
|
||
${tlsDir} = {
|
||
hostPath = tlsDir;
|
||
isReadOnly = true;
|
||
};
|
||
};
|
||
config =
|
||
{ ... }:
|
||
{
|
||
imports = [
|
||
../container-modules/swarm-container.nix
|
||
(import ../container-modules/swarm-container-resolver.nix {
|
||
inherit (networkCfg) bridgeIp;
|
||
# The responder introspects authelia BY NAME on every auth
|
||
# request, and a responder that cannot resolve it denies
|
||
# every client — so it must not start before the resolver
|
||
# file exists.
|
||
dnsConsumers = [ "swarm-nats-auth.service" ];
|
||
})
|
||
caBundleModule
|
||
];
|
||
|
||
services.hyperhive.swarmContainer = { inherit privateNetwork; };
|
||
|
||
services.nats = {
|
||
enable = true;
|
||
|
||
# Retention, so a reader can ask "what did this hive last
|
||
# say?" without anyone keeping a second copy. The upstream
|
||
# option also wires `settings.jetstream.store_dir = dataDir`;
|
||
# the container is `ephemeral = false`, so that survives a
|
||
# restart with no bind mount.
|
||
#
|
||
# ⚠️ Losing the store is not a correctness problem here: a
|
||
# reader then sees nothing for every hive, which is the true
|
||
# answer until each one publishes again. It degrades to
|
||
# honesty rather than to a stale "healthy".
|
||
jetstream = true;
|
||
|
||
serverName = "swarm-nats";
|
||
port = cfg.port;
|
||
# Off. The `tls` block below names a leaf that exists only at
|
||
# runtime, and `nats-server -t` loads it, so the build-time check
|
||
# fails on every hive; so do the callout keys, empty in
|
||
# `settings`. Upstream's own description names the case: disable
|
||
# it when the config includes other files. The check moves to
|
||
# server start.
|
||
validateConfig = false;
|
||
|
||
settings =
|
||
calloutBlocks {
|
||
userKey = "";
|
||
issuerKey = "";
|
||
}
|
||
// {
|
||
# The most a single publish may be, in bytes, before the
|
||
# server answers `-ERR 'Maximum Payload Violation'` and
|
||
# closes the connection — not a truncation, a dropped row
|
||
# and a reconnect. This queue carries agent terminal rows
|
||
# published whole rather than split, so upstream's own
|
||
# default is sized to lose one of those rather than merely
|
||
# shorten it. Bounded from above by
|
||
# `max_pending`: nats-server refuses to start once this
|
||
# exceeds it, and widening that ceiling instead costs
|
||
# memory per connection, so this stays comfortably under it.
|
||
max_payload = 8388608;
|
||
|
||
# The monitoring endpoint, which is what the exporter below
|
||
# reads. `//` adds a key `calloutBlocks` does not produce
|
||
# (`accounts`, `authorization`) — checked, because a shallow
|
||
# merge that collided here would drop the auth config while
|
||
# rendering a config the server starts on.
|
||
#
|
||
# ⚠️ Loopback: unauthenticated, and `/connz` lists every
|
||
# connected client.
|
||
http = "127.0.0.1:${toString cfg.monitorPort}";
|
||
|
||
# TLS required on the client port: a server with a `tls`
|
||
# block advertises `tls_required` and refuses a client that
|
||
# does not upgrade. No `allow_non_tls`, which would keep the
|
||
# plaintext path open across the mesh. The 2s timeout is for
|
||
# a handshake that crosses the mesh; upstream's 0.5s is sized
|
||
# for a LAN.
|
||
tls = {
|
||
cert_file = tlsCertPath;
|
||
key_file = tlsKeyCredentialPath;
|
||
timeout = 2;
|
||
};
|
||
};
|
||
};
|
||
|
||
# The key as the `nats` user can read it; see `tlsDir`. Resolved at
|
||
# unit start only, which is why a renewed leaf restarts the server
|
||
# rather than reloading it.
|
||
systemd.services.nats.serviceConfig.LoadCredential = [
|
||
"${tlsKeyCredential}:${tlsKeyPath}"
|
||
];
|
||
|
||
# The translation layer. NATS has no Prometheus format of its
|
||
# own, so this reads the JSON monitoring endpoint above and
|
||
# re-serves it in the format the collector scrapes.
|
||
#
|
||
# Upstream's exporter module rather than a hand-rolled unit, for
|
||
# the same reason `services.nats`'s own `settings` is kept: the
|
||
# reviewed reasoning about flags and hardening lives there.
|
||
services.prometheus.exporters.nats = {
|
||
enable = true;
|
||
listenAddress = "127.0.0.1";
|
||
port = cfg.metricsPort;
|
||
url = "http://127.0.0.1:${toString cfg.monitorPort}";
|
||
|
||
# ⚠️ NOT optional, despite the name. Upstream renders
|
||
# `-addr … -port … ${extraFlags} ${url}` and defaults
|
||
# `extraFlags` to the empty list, but the exporter REFUSES TO
|
||
# START without at least one collector: it exits 1 with
|
||
# "no Collectors specified". Shipped without this the unit logs
|
||
# `Started`, the process is gone milliseconds later, and the
|
||
# scrape is refused — measured, and invisible to evaluation
|
||
# because the empty list renders perfectly.
|
||
#
|
||
# Which three, and why not more: `varz` is the server itself
|
||
# (uptime, memory, slow consumers), `connz` is per-connection so
|
||
# a client that will not stay connected is visible, and `jsz`
|
||
# covers JetStream, which this swarm relies on for the status
|
||
# KV. The remaining collectors describe a clustered deployment
|
||
# this swarm does not have.
|
||
extraFlags = [
|
||
"-varz"
|
||
"-connz"
|
||
"-jsz=all"
|
||
];
|
||
};
|
||
|
||
# A wrapper that includes upstream's rendered settings verbatim
|
||
# plus the runtime fragment; rendering the config ourselves
|
||
# instead would throw away upstream's `settings`, where the
|
||
# reviewed reasoning lives. `renderConfig` writes it at every start,
|
||
# because the includes must be siblings of the fragment (see
|
||
# `runtimeDir`).
|
||
#
|
||
# `mkForce`: upstream defines ExecStart inside an `mkMerge`, so a
|
||
# plain override conflicts rather than wins.
|
||
systemd.services.nats.serviceConfig.ExecStart =
|
||
lib.mkForce "${pkgs.nats-server}/bin/nats-server -c ${runtimeWrapper}";
|
||
systemd.services.nats.serviceConfig.ExecStartPre = "+${renderConfig}";
|
||
|
||
# The auth-callout responder: the half that lets the server
|
||
# above say *yes*. Without it the `auth_callout` block is a
|
||
# door nobody can open, which is the deliberate interim state.
|
||
systemd.services.swarm-nats-auth = {
|
||
description = "swarm queue auth-callout responder";
|
||
after = [ "nats.service" ];
|
||
requires = [ "nats.service" ];
|
||
# `nats.service` too: a server started late, once its keys arrived,
|
||
# brings the responder up with it.
|
||
wantedBy = [
|
||
"multi-user.target"
|
||
"nats.service"
|
||
];
|
||
serviceConfig = {
|
||
ExecStart = lib.concatStringsSep " " [
|
||
"${deployCfg.nats.authPackage}/bin/swarm-nats-auth"
|
||
# By name, like every other client: the leaf carries the name
|
||
# and no IP, so a loopback address fails verification. The
|
||
# container's resolver goes through the bridge, where dnsmasq
|
||
# answers it.
|
||
"--nats-url tls://${cfg.domain}:${toString cfg.port}"
|
||
"--client-id ${lib.escapeShellArg cfg.clientId}"
|
||
# The account admitted clients land in, by NAME: in
|
||
# server-config mode the server resolves `aud` against its
|
||
# own `accounts` block, so this and the block above have to
|
||
# be the same string — which is why both come from one let.
|
||
"--account ${lib.escapeShellArg clientAccount}"
|
||
"--introspection-url ${lib.escapeShellArg introspectionUrl}"
|
||
# Each of these names a principal some OTHER module mints,
|
||
# so each is read out of that module rather than spelled
|
||
# again here — same argument as `--account` above, one
|
||
# level wider. The responder denies a client id it does
|
||
# not recognise, and a NATS denial arrives as a timeout,
|
||
# so a drift here is silent at the point of change and
|
||
# misattributed at the point of failure.
|
||
"--hive-client-prefix ${lib.escapeShellArg autheliaCfg.hiveClientPrefix}"
|
||
"--agent-client-suffix ${lib.escapeShellArg autheliaCfg.agentClientSuffix}"
|
||
"--reader-client ${lib.escapeShellArg controllerCfg.queueClientId}"
|
||
# Hears try-outs and answers each by reply, and nothing else.
|
||
"--review-classifier-client ${lib.escapeShellArg config.services.hyperhive.swarm.review-classifier.clientId}"
|
||
# Hears builds and answers each by reply, and nothing else.
|
||
"--builder-client ${lib.escapeShellArg config.services.hyperhive.swarm.builder.clientId}"
|
||
# Hears login steps and answers each by reply, and nothing else.
|
||
"--claude-login-client ${lib.escapeShellArg config.services.hyperhive.swarm.claude-login.clientId}"
|
||
# What an agent may publish to, keyed on its own name: its
|
||
# terminal rows and its turn-state header
|
||
# (`hive-agent::swarm_agent_state`), two subject families so a
|
||
# subscriber following one agent's terminal is not also handed
|
||
# every header. The hive's shared agent client gets nothing.
|
||
#
|
||
# `$$`, not `$`: systemd substitutes `$NAME` in `ExecStart`
|
||
# whether or not it is quoted, so a single dollar reaches the
|
||
# responder as the empty expansion of an unset `SWARM`.
|
||
"--agent-token-publish-subject ${lib.escapeShellArg "\$\$SWARM.term.{agent}"}"
|
||
"--agent-token-publish-subject ${lib.escapeShellArg "\$\$SWARM.agent-state.{agent}"}"
|
||
# Its own key in the `agent-icons` KV bucket, and nothing else
|
||
# of JetStream. A KV put is a JetStream publish to
|
||
# `$KV.<bucket>.<key>`, acked on the client's inbox, so this
|
||
# one subject is the whole write. The harness publishes to the
|
||
# subject directly instead of opening the bucket, so it needs
|
||
# no `$JS.API.*` subject; swarm-controller creates the bucket.
|
||
"--agent-token-publish-subject ${lib.escapeShellArg "\$\$KV.agent-icons.{agent}"}"
|
||
# Its subagents' terminal rows
|
||
# (`swarm_queue_client::subagent_term`), publish only and no
|
||
# `$JS.API.*` subject: swarm-controller creates the
|
||
# `term-sub-{agent}` stream that keeps them, so its subjects
|
||
# and limits are never the agent's to choose.
|
||
"--agent-token-publish-subject ${lib.escapeShellArg "\$\$SWARM.term.{agent}.sub.>"}"
|
||
# Reading its own key of the `agent-schedules` KV bucket
|
||
# (`swarm_queue_client::agent_schedules`), and nothing else of
|
||
# JetStream: a direct get names the key in its subject, so
|
||
# this one subject is one key. No `STREAM.INFO`, no
|
||
# `CONSUMER.CREATE` (a watch's create subject carries no key,
|
||
# so it would cover every agent's) and no `$KV` write.
|
||
"--agent-token-publish-subject ${lib.escapeShellArg "\$\$JS.API.DIRECT.GET.KV_agent-schedules.\$\$KV.agent-schedules.{agent}"}"
|
||
# Its own key of the `agent-inference` KV bucket
|
||
# (`swarm_queue_client::agent_inference`), on the same terms.
|
||
"--agent-token-publish-subject ${lib.escapeShellArg "\$\$JS.API.DIRECT.GET.KV_agent-inference.\$\$KV.agent-inference.{agent}"}"
|
||
"--store-cert-role ${lib.escapeShellArg authCertRole}"
|
||
];
|
||
# The store identity arrives by `LoadCredential` and is named
|
||
# only as a **path**: `argv` is world-readable via
|
||
# /proc/<pid>/cmdline. Every secret the responder holds beyond
|
||
# it, its own OIDC client secret included, it reads from the
|
||
# store with that identity.
|
||
LoadCredential = lib.mapAttrsToList (id: _: "${id}:${inContainer id}") authStoreFiles;
|
||
DynamicUser = true;
|
||
Restart = "on-failure";
|
||
RestartSec = "5s";
|
||
SyslogIdentifier = "swarm-nats-auth";
|
||
};
|
||
# The store the responder reads the callout keys and agent
|
||
# credentials from.
|
||
environment = {
|
||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||
BAO_CLIENT_CERT = authCredential "bao-client.pem";
|
||
BAO_CLIENT_KEY = authCredential "bao-client-key.pem";
|
||
}
|
||
// lib.optionalAttrs (authStoreFiles ? "bao-ca.pem") {
|
||
BAO_CACERT = authCredential "bao-ca.pem";
|
||
};
|
||
};
|
||
|
||
# The server binary, so an operator with a shell in here can
|
||
# run `nats-server -t` against the generated config. The unit
|
||
# resolves ExecStart through the store path and puts nothing
|
||
# on PATH.
|
||
environment.systemPackages = [ pkgs.nats-server ];
|
||
};
|
||
};
|
||
|
||
# Order the container after the host CA generator, so the bind source
|
||
# exists before nspawn sets the mount up. Without it a late CA fails the
|
||
# container start outright rather than degrading.
|
||
systemd.services."container@${machine}" = caTrust.containerOrdering;
|
||
|
||
# Deliver the responder's store identity into the container before it
|
||
# starts. A copy rather than a `bindMounts` entry: nixos-container refuses
|
||
# to start when a bind source is missing, so one absent file would take
|
||
# down the **whole container including the queue**, not merely the
|
||
# responder.
|
||
systemd.services.swarm-nats-auth-secrets = {
|
||
description = "deliver the swarm queue responder's credentials";
|
||
before = [ "container@swarm-nats.service" ];
|
||
wantedBy = [ "container@swarm-nats.service" ];
|
||
# Where the store's PKI is minted on this host, the responder's leaf is
|
||
# one of its files, and the minting unit runs before every copy. Ordering
|
||
# alone is not enough: the queue's container, which pulls this unit in,
|
||
# starts from `machines.target`, outside the transaction that runs the
|
||
# minting unit from `multi-user.target`. Ignored elsewhere, where the unit
|
||
# does not exist.
|
||
after = [ "swarm-bao-pki.service" ];
|
||
wants = [ "swarm-bao-pki.service" ];
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
SyslogIdentifier = "swarm-nats-auth-secrets";
|
||
};
|
||
path = [ pkgs.coreutils ];
|
||
script = ''
|
||
set -euo pipefail
|
||
install -d -m 0700 ${lib.escapeShellArg secretDir}
|
||
''
|
||
+ lib.concatStrings (
|
||
lib.mapAttrsToList (id: source: ''
|
||
install -m 0400 ${lib.escapeShellArg source} ${lib.escapeShellArg (hostPath id)}
|
||
'') authStoreFiles
|
||
);
|
||
};
|
||
|
||
# The callout's three public keys, from the store into the files
|
||
# `renderConfig` reads at every server start. `ensure-callout-keys` first
|
||
# creates each pair the store holds none of; it needs no queue, so on a
|
||
# first boot the keys exist before the server that requires them, and
|
||
# nothing here waits on the responder, which itself waits on the server.
|
||
#
|
||
# An issuer or user seed file left on this host is what `ensure-callout-keys`
|
||
# stores for that key when the store holds none, so the key keeps its
|
||
# public half. It removes the file once a read of the store returns a seed,
|
||
# in this run and never before: a removal that ran ahead of the import,
|
||
# with the store sealed, would leave the next run to mint a new key.
|
||
#
|
||
# Before the container and wanted by it, the shape `swarm-bao-nats-tls`
|
||
# has: a store that comes up later fails this run, the server refuses to
|
||
# start without the files (or starts with the ones last fetched), and the
|
||
# retry that lands them starts or restarts the server. Under the
|
||
# responder's own leaf, the one principal that may create the keys' paths.
|
||
systemd.services.swarm-bao-nats-callout-xkey = {
|
||
description = "create the swarm queue's callout keys if absent and fetch their public halves for the server";
|
||
# Ordered after the store where it runs here
|
||
# (./glue-bao-readers-store-order.nix); elsewhere the timeout below
|
||
# bounds the attempt.
|
||
before = [ "container@${machine}.service" ];
|
||
wantedBy = [
|
||
"multi-user.target"
|
||
"container@${machine}.service"
|
||
];
|
||
path = [
|
||
deployCfg.nats.authPackage
|
||
baoDeploy.package
|
||
pkgs.coreutils
|
||
pkgs.systemd
|
||
];
|
||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||
serviceConfig = storeRetry.serviceConfig // {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
SyslogIdentifier = "swarm-bao-nats-callout-xkey";
|
||
TimeoutStartSec = 30;
|
||
};
|
||
environment = {
|
||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||
BAO_CLIENT_CERT = deployCfg.nats.authBaoClientCertFile;
|
||
BAO_CLIENT_KEY = deployCfg.nats.authBaoClientKeyFile;
|
||
}
|
||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||
BAO_CACERT = baoDeploy.serverCaFile;
|
||
};
|
||
script = ''
|
||
set -euo pipefail
|
||
|
||
${atomicWriteSecret}
|
||
${refreshConsumer}
|
||
|
||
swarm-nats-auth ensure-callout-keys --store-cert-role ${lib.escapeShellArg authCertRole} \
|
||
--import-issuer-seed-file ${lib.escapeShellArg importIssuerSeed} \
|
||
--import-user-seed-file ${lib.escapeShellArg importUserSeed}
|
||
if [ -d ${lib.escapeShellArg importSeedDir} ]; then
|
||
rmdir --ignore-fail-on-non-empty ${lib.escapeShellArg importSeedDir}
|
||
fi
|
||
|
||
err="$(mktemp)"
|
||
trap 'rm -f "$err"' EXIT
|
||
|
||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||
echo "could not log in to swarm-bao with the queue responder's certificate; leaving the callout key files as they are." >&2
|
||
cat "$err" >&2
|
||
exit 1
|
||
fi
|
||
export BAO_TOKEN
|
||
|
||
install -d -m 0755 ${lib.escapeShellArg hostRuntimeDir}
|
||
''
|
||
+ lib.concatMapStrings (
|
||
k:
|
||
let
|
||
file = "${hostRuntimeDir}/${k.file}";
|
||
in
|
||
''
|
||
if ! key="$(bao kv get -field=value ${lib.escapeShellArg k.storePath} 2>"$err")"; then
|
||
echo "swarm-bao did not return ${k.storePath}." >&2
|
||
cat "$err" >&2
|
||
exit 1
|
||
fi
|
||
if [ -z "$key" ]; then
|
||
echo "swarm-bao returned an empty ${k.storePath}." >&2
|
||
exit 1
|
||
fi
|
||
if secret_differs ${lib.escapeShellArg file} "$key"; then
|
||
atomic_write_secret 0444 "" ${lib.escapeShellArg file} "$key"
|
||
fi
|
||
''
|
||
) calloutPublicKeys
|
||
# `renderConfig` reads the files as the server starts, and only then.
|
||
+ lib.concatMapStrings (k: ''
|
||
refresh_consumer ${lib.escapeShellArg machine} nats.service ${lib.escapeShellArg "${hostRuntimeDir}/${k.file}"}
|
||
'') calloutPublicKeys;
|
||
};
|
||
|
||
# The queue's TLS leaf, issued by the secret store's `pki/roles/swarm-nats`
|
||
# under this host's `swarm-nats` identity. Same shape as ./hive-tls.nix's
|
||
# `swarm-services-cert`, whose comments carry the reasoning for the login,
|
||
# the single `issue` call split with `jq`, and the unseal-sized retry.
|
||
#
|
||
# Before the container, so a normal boot starts the server with its leaf.
|
||
# A store that comes up later fails this run; the retry lands the leaf and
|
||
# restarts the server, which until then refuses to start (TLS required,
|
||
# and its key credential is missing).
|
||
systemd.services.swarm-bao-nats-tls = {
|
||
description = "Issue the swarm queue's TLS leaf from the secret store's PKI";
|
||
wantedBy = [
|
||
"multi-user.target"
|
||
"container@${machine}.service"
|
||
];
|
||
before = [ "container@${machine}.service" ];
|
||
# Both absent where the store runs elsewhere, and ignored there; see
|
||
# `swarm-services-cert`. The ordering after this leaf's policy unit is
|
||
# ./glue-bao-readers-policy-order.nix's, because it holds only where the
|
||
# store is here too.
|
||
after = [
|
||
"container@${baoCfg.machine}.service"
|
||
"swarm-bao-pki.service"
|
||
];
|
||
wants = [ "container@${baoCfg.machine}.service" ];
|
||
path = [
|
||
baoDeploy.package
|
||
pkgs.jq
|
||
pkgs.openssl
|
||
pkgs.coreutils
|
||
pkgs.gnugrep
|
||
pkgs.systemd
|
||
];
|
||
startLimitBurst = 2880;
|
||
startLimitIntervalSec = 90000;
|
||
# No `RemainAfterExit`: the timer below starts this again, and starting
|
||
# an active unit is a no-op.
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
UMask = "0077";
|
||
SyslogIdentifier = "swarm-bao-nats-tls";
|
||
Restart = "on-failure";
|
||
RestartSec = 30;
|
||
# An attempt that rotates the leaf waits on a blocking restart of the
|
||
# server in its container: up to 150s to stop (nixpkgs' nats module)
|
||
# and 90s to start.
|
||
TimeoutStartSec = "5min";
|
||
};
|
||
environment = {
|
||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||
}
|
||
// lib.optionalAttrs (deployCfg.nats.baoClientCertFile != null) {
|
||
BAO_CLIENT_CERT = deployCfg.nats.baoClientCertFile;
|
||
}
|
||
// lib.optionalAttrs (deployCfg.nats.baoClientKeyFile != null) {
|
||
BAO_CLIENT_KEY = deployCfg.nats.baoClientKeyFile;
|
||
}
|
||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||
BAO_CACERT = baoDeploy.serverCaFile;
|
||
};
|
||
script = ''
|
||
set -euo pipefail
|
||
d=${lib.escapeShellArg tlsDir}
|
||
name=${lib.escapeShellArg cfg.domain}
|
||
install -d -m 0755 "$d"
|
||
|
||
# Missing, past half its window, or naming something other than the
|
||
# configured domain.
|
||
reissue=0
|
||
{ [ -s "$d/cert.pem" ] && [ -s "$d/key.pem" ]; } || reissue=1
|
||
openssl x509 -in "$d/cert.pem" -noout -checkend ${toString leafRenewSeconds} >/dev/null 2>&1 || reissue=1
|
||
openssl x509 -in "$d/cert.pem" -noout -checkhost "$name" 2>/dev/null | grep -q ' does match ' || reissue=1
|
||
|
||
if [ "$reissue" = 0 ]; then
|
||
echo "queue leaf valid for $name — leaving it alone"
|
||
exit 0
|
||
fi
|
||
|
||
${
|
||
if deployCfg.nats.baoClientCertFile == null || deployCfg.nats.baoClientKeyFile == null then
|
||
''
|
||
echo "no bao client certificate configured for the queue, so its TLS leaf" >&2
|
||
echo "cannot be requested from the store." >&2
|
||
echo "Set services.hyperhive.deploy.nats.baoClient{Cert,Key}File" >&2
|
||
echo "to a leaf the store's CA signed with CN=${baoDeploy.natsCommonName}." >&2
|
||
echo "They default to the leaf the store mints only where" >&2
|
||
echo "services.hyperhive.deploy.bao.enable is set on this host." >&2
|
||
exit 1''
|
||
else
|
||
""
|
||
}
|
||
|
||
err="$(mktemp)"
|
||
trap 'rm -f "$err"' EXIT
|
||
|
||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||
echo "could not log in to the swarm secret store with the queue's certificate." >&2
|
||
cat "$err" >&2
|
||
exit 1
|
||
fi
|
||
export BAO_TOKEN
|
||
|
||
echo "requesting the queue's TLS leaf for $name from the store"
|
||
resp="$(mktemp "$d/issue.json.XXXXXX")"
|
||
trap 'rm -f "$err" "$resp"' EXIT
|
||
|
||
if ! bao write -format=json \
|
||
${lib.escapeShellArg "${baoDeploy.servicesPkiMountPath}/issue/${baoDeploy.natsPkiRoleName}"} \
|
||
common_name="$name" > "$resp" 2>"$err"; then
|
||
echo "the store refused to issue the queue's certificate." >&2
|
||
cat "$err" >&2
|
||
exit 1
|
||
fi
|
||
|
||
jq -r '.data.private_key // empty' < "$resp" > "$d/key.pem.new"
|
||
jq -r '.data.certificate // empty' < "$resp" > "$d/cert.pem.new"
|
||
for f in "$d/key.pem.new" "$d/cert.pem.new"; do
|
||
if [ ! -s "$f" ]; then
|
||
echo "the store's response was missing a field: $f is empty" >&2
|
||
exit 1
|
||
fi
|
||
done
|
||
chmod 0600 "$d/key.pem.new"
|
||
chmod 0644 "$d/cert.pem.new"
|
||
mv -f "$d/key.pem.new" "$d/key.pem"
|
||
mv -f "$d/cert.pem.new" "$d/cert.pem"
|
||
${lib.optionalString gatewayServesQueueLeaf ''
|
||
|
||
# The gateway's `/metrics` vhost serves a copy of this leaf.
|
||
systemctl start swarm-nats-gateway-cert.service''}
|
||
|
||
# Renewal, or the late-store retry. On a normal boot the container is
|
||
# not up yet and starts the server with this leaf itself. A restart,
|
||
# not a reload: the key arrives by `LoadCredential`, which systemd
|
||
# resolves at start only. `reset-failed` because a server that found
|
||
# no key may have hit its start limit.
|
||
if systemctl is-active --quiet ${lib.escapeShellArg "container@${machine}.service"}; then
|
||
echo "queue leaf rotated — restarting the server in ${machine}"
|
||
systemctl --machine=${lib.escapeShellArg machine} reset-failed nats.service
|
||
systemctl --machine=${lib.escapeShellArg machine} restart nats.service
|
||
fi
|
||
'';
|
||
};
|
||
|
||
systemd.timers.swarm-bao-nats-tls = {
|
||
description = "Daily renewal check for the swarm queue's TLS leaf";
|
||
wantedBy = [ "timers.target" ];
|
||
timerConfig = {
|
||
OnCalendar = "daily";
|
||
Persistent = true;
|
||
};
|
||
};
|
||
};
|
||
}
|