swarm-nats: fetch the callout xkey before the server starts and seal callout traffic
swarm-bao-nats-callout-xkey runs on the queue's host before the swarm-nats container, under the responder's store leaf: it runs `swarm-nats-auth ensure-xkey`, reads the public half back from the store, and writes it where the container's runtime config dir is. On a change it restarts the server. nats-server now always starts through the runtime wrapper, in manual mode too. A privileged ExecStartPre renders auth_callout with the xkey (and, in auto mode, the two generated nkeys) at every start, and fails the server when no xkey has been fetched. The auto-mode key unit now only mints seeds and writes their public halves. The responder is wanted by nats.service, so a server started late, once its key arrived, brings the responder up with it.
This commit is contained in:
atlas
mara
parent
bb571285fd
commit
9a241a1df1
4 changed files with 362 additions and 48 deletions
|
|
@ -57,6 +57,11 @@ let
|
|||
here = deployCfg.nats.enable;
|
||||
policy = "swarm-bao-nats-auth-policy";
|
||||
};
|
||||
# ./swarm-nats.nix, under the same leaf.
|
||||
swarm-bao-nats-callout-xkey = {
|
||||
here = deployCfg.nats.enable;
|
||||
policy = "swarm-bao-nats-auth-policy";
|
||||
};
|
||||
# ./hive-matrix.nix, under matrix-ctl's leaf.
|
||||
swarm-bao-matrix-oidc = {
|
||||
here = deployCfg.matrix.enable;
|
||||
|
|
|
|||
|
|
@ -94,25 +94,27 @@ let
|
|||
|
||||
# Upstream's rendered settings, re-rendered with the same generator on the
|
||||
# same value — the artifact `services.nats` would have used, not a
|
||||
# transcription. ⚠️ This reference is also the only thing keeping it alive:
|
||||
# `ExecStart` names a runtime path, so nothing else in the closure names
|
||||
# the store file, and its string context is what stops it being
|
||||
# garbage-collected out from under a running server.
|
||||
# transcription. ⚠️ `renderConfig` below is the only thing naming it:
|
||||
# `ExecStart` names a runtime path, so that reference is what keeps the store
|
||||
# file in the closure and stops it being garbage-collected out from under a
|
||||
# running server.
|
||||
renderedSettings = natsFormat.generate "nats.conf" (
|
||||
config.containers.${machine}.config.services.nats.settings
|
||||
);
|
||||
|
||||
# Defined once, rendered twice: into `settings` with the operator's values,
|
||||
# and into the runtime fragment with placeholders the generator fills in.
|
||||
# and into the runtime fragment with placeholders filled in at server start
|
||||
# (the xkey always, the two nkeys in auto mode).
|
||||
#
|
||||
# 🪤 Do not inline these and hand-write the fragment instead. Both files are
|
||||
# loaded and the fragment is the LATER definition, so it wins — a
|
||||
# hand-written copy would make a future edit to `settings` silently
|
||||
# ineffective on exactly the hives that use auto mode.
|
||||
# ineffective.
|
||||
calloutBlocks =
|
||||
{
|
||||
userKey,
|
||||
issuerKey,
|
||||
xkey ? null,
|
||||
}:
|
||||
{
|
||||
# Two accounts, and the callout user lives in neither of the accounts
|
||||
|
|
@ -191,7 +193,10 @@ let
|
|||
issuer = issuerKey;
|
||||
auth_users = [ userKey ];
|
||||
account = calloutAccount;
|
||||
};
|
||||
}
|
||||
# The curve key the server seals each request to, and opens the
|
||||
# reply with. A key that does not parse stops the server starting.
|
||||
// lib.optionalAttrs (xkey != null) { inherit xkey; };
|
||||
};
|
||||
};
|
||||
|
||||
|
|
@ -200,10 +205,85 @@ let
|
|||
# fragment is generated rather than transcribed — NATS' config parser
|
||||
# accepts JSON, and an `include` of it merges (measured).
|
||||
calloutTemplate = natsFormat.generate "swarm-nats-callout-template.conf" (calloutBlocks {
|
||||
userKey = "@USER_PUBKEY@";
|
||||
issuerKey = "@ISSUER_PUBKEY@";
|
||||
userKey =
|
||||
if deployCfg.nats.autoGenerateCallout then "@USER_PUBKEY@" else deployCfg.nats.calloutUserPublicKey;
|
||||
issuerKey =
|
||||
if deployCfg.nats.autoGenerateCallout then
|
||||
"@ISSUER_PUBKEY@"
|
||||
else
|
||||
deployCfg.nats.calloutIssuerPublicKey;
|
||||
xkey = "@XKEY_PUBKEY@";
|
||||
});
|
||||
|
||||
# Where the fetch unit leaves the xkey's public half, read from the store at
|
||||
# `xkeyPublicPath`. In auto mode the nkeys' public halves sit beside it.
|
||||
xkeyPublicFile = "callout-xkey.pub";
|
||||
userPublicFile = "callout-user.pub";
|
||||
issuerPublicFile = "issuer.pub";
|
||||
|
||||
# The callout xkey's public half in the store, readable by any token. The
|
||||
# Rust half is `swarm-nats-auth`'s `xkey::PUBLIC_PATH`; the seed beside it
|
||||
# is `xkeySeedLeaf` in ./swarm-bao.nix.
|
||||
xkeyPublicPath = "secret/swarm/public/swarm-nats/callout-xkey";
|
||||
|
||||
# Each placeholder in `calloutTemplate`, as a `sed` expression over the
|
||||
# shell variable `renderConfig` reads its value into.
|
||||
calloutSubstitutions = [
|
||||
''-e "s|@XKEY_PUBKEY@|$xkey|g"''
|
||||
]
|
||||
++ lib.optionals deployCfg.nats.autoGenerateCallout [
|
||||
''-e "s|@USER_PUBKEY@|$user_pub|g"''
|
||||
''-e "s|@ISSUER_PUBKEY@|$issuer_pub|g"''
|
||||
];
|
||||
|
||||
# Renders the server's config as it starts, so the settings it includes are
|
||||
# always the running system's. Exits non-zero, and so stops the server
|
||||
# starting, while no xkey has been fetched: the server never runs without
|
||||
# one. `+` because the server's own sandbox cannot write `runtimeDir`.
|
||||
renderConfig = pkgs.writeShellScript "swarm-nats-render-config" ''
|
||||
set -euo pipefail
|
||||
PATH=${
|
||||
lib.makeBinPath [
|
||||
pkgs.coreutils
|
||||
pkgs.gnugrep
|
||||
pkgs.gnused
|
||||
]
|
||||
}
|
||||
d=${lib.escapeShellArg runtimeDir}
|
||||
|
||||
if [ ! -s "$d/${xkeyPublicFile}" ]; then
|
||||
echo "${runtimeDir}/${xkeyPublicFile} is missing: swarm-bao-nats-callout-xkey on the host fetches it from the secret store, and the queue does not start without it" >&2
|
||||
exit 1
|
||||
fi
|
||||
xkey="$(cat "$d/${xkeyPublicFile}")"
|
||||
# A public curve key is X and 55 base32 characters; checked here because
|
||||
# it is about to become part of a sed expression.
|
||||
if ! printf '%s' "$xkey" | grep -Eqx 'X[A-Z2-7]{55}'; then
|
||||
echo "${runtimeDir}/${xkeyPublicFile} does not hold a public xkey" >&2
|
||||
exit 1
|
||||
fi
|
||||
${lib.optionalString deployCfg.nats.autoGenerateCallout ''
|
||||
user_pub="$(cat "$d/${userPublicFile}")"
|
||||
issuer_pub="$(cat "$d/${issuerPublicFile}")"
|
||||
''}
|
||||
sed ${lib.concatStringsSep " " calloutSubstitutions} ${calloutTemplate} > "$d/callout.conf.tmp"
|
||||
# Mode BEFORE the rename: a rename publishes whatever the file already
|
||||
# is. World-readable, because the server reads it as `nats` and
|
||||
# everything in it is public.
|
||||
chmod 0444 "$d/callout.conf.tmp"
|
||||
mv "$d/callout.conf.tmp" "$d/callout.conf"
|
||||
|
||||
# Upstream's rendered settings, as a sibling the wrapper can name without
|
||||
# a leading slash.
|
||||
ln -sfn ${renderedSettings} "$d/settings.conf"
|
||||
|
||||
# The wrapper. Bare filenames — see `runtimeDir`. printf rather than a
|
||||
# heredoc, whose terminator would depend on nix's indentation stripping.
|
||||
printf 'include "settings.conf"\ninclude "callout.conf"\n' > "$d/nats.conf.tmp"
|
||||
chmod 0444 "$d/nats.conf.tmp"
|
||||
mv "$d/nats.conf.tmp" "$d/nats.conf"
|
||||
'';
|
||||
|
||||
baoCfg = config.services.hyperhive.swarm.bao;
|
||||
baoDeploy = deployCfg.bao;
|
||||
|
||||
|
|
@ -305,9 +385,9 @@ in
|
|||
them from `calloutUserPublicKey` / `calloutIssuerPublicKey`.
|
||||
|
||||
A first-boot unit mints both keypairs if absent, keeps the seeds
|
||||
host-side at `0600`, and writes only the public halves into a
|
||||
fragment the server reads. Nothing secret is evaluated, so
|
||||
nothing secret reaches the nix store.
|
||||
host-side at `0600`, and writes only the public halves where the
|
||||
server's config is rendered from as it starts. Nothing secret is
|
||||
evaluated, so nothing secret reaches the nix store.
|
||||
|
||||
Leave it off wherever the queue and its clients are not the same
|
||||
operator's problem: the seeds must reach whoever runs the
|
||||
|
|
@ -744,14 +824,15 @@ in
|
|||
# A wrapper that includes upstream's rendered settings verbatim
|
||||
# plus the runtime fragment; rendering the config ourselves
|
||||
# instead would throw away upstream's `settings`, where the
|
||||
# reviewed reasoning lives. The generator writes it, because the
|
||||
# includes must be siblings of the fragment (see `runtimeDir`).
|
||||
# reviewed reasoning lives. `renderConfig` writes it at every start,
|
||||
# because the includes must be siblings of the fragment (see
|
||||
# `runtimeDir`).
|
||||
#
|
||||
# `mkForce`: upstream defines ExecStart inside an `mkMerge`, so a
|
||||
# plain override conflicts rather than wins.
|
||||
systemd.services.nats.serviceConfig.ExecStart = lib.mkIf deployCfg.nats.autoGenerateCallout (
|
||||
lib.mkForce "${pkgs.nats-server}/bin/nats-server -c ${runtimeWrapper}"
|
||||
);
|
||||
systemd.services.nats.serviceConfig.ExecStart =
|
||||
lib.mkForce "${pkgs.nats-server}/bin/nats-server -c ${runtimeWrapper}";
|
||||
systemd.services.nats.serviceConfig.ExecStartPre = "+${renderConfig}";
|
||||
|
||||
# The auth-callout responder: the half that lets the server
|
||||
# above say *yes*. Without it the `auth_callout` block is a
|
||||
|
|
@ -766,7 +847,12 @@ in
|
|||
description = "swarm queue auth-callout responder";
|
||||
after = [ "nats.service" ];
|
||||
requires = [ "nats.service" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
# `nats.service` too: a server started late, once its xkey arrived,
|
||||
# brings the responder up with it.
|
||||
wantedBy = [
|
||||
"multi-user.target"
|
||||
"nats.service"
|
||||
];
|
||||
serviceConfig = {
|
||||
ExecStart = lib.concatStringsSep " " [
|
||||
"${deployCfg.nats.authPackage}/bin/swarm-nats-auth"
|
||||
|
|
@ -1030,6 +1116,100 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
# The callout xkey's public half, from the store into the file
|
||||
# `renderConfig` reads at every server start. `ensure-xkey` first creates
|
||||
# the pair when the store holds none; it needs no queue, so on a first boot
|
||||
# the key exists before the server that requires it, and nothing here
|
||||
# waits on the responder, which itself waits on the server.
|
||||
#
|
||||
# Before the container and wanted by it, the shape `swarm-bao-nats-tls`
|
||||
# has: a store that comes up later fails this run, the server refuses to
|
||||
# start without the file (or starts with the one last fetched), and the
|
||||
# retry that lands it starts or restarts the server. Under the responder's
|
||||
# own leaf, the one principal that may create the two xkey paths.
|
||||
systemd.services.swarm-bao-nats-callout-xkey = {
|
||||
description = "create the swarm queue's callout xkey if absent and fetch its public half for the server";
|
||||
# As `swarm-bao-nats-auth-oidc`: the store's units exist only where it
|
||||
# runs.
|
||||
after = lib.optionals baoDeploy.enable [
|
||||
"swarm-bao-pki.service"
|
||||
"container@${baoCfg.machine}.service"
|
||||
];
|
||||
wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ];
|
||||
requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ];
|
||||
before = [ "container@${machine}.service" ];
|
||||
wantedBy = [
|
||||
"multi-user.target"
|
||||
"container@${machine}.service"
|
||||
];
|
||||
path = [
|
||||
deployCfg.nats.authPackage
|
||||
baoDeploy.package
|
||||
pkgs.coreutils
|
||||
pkgs.systemd
|
||||
];
|
||||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||||
serviceConfig = storeRetry.serviceConfig // {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
SyslogIdentifier = "swarm-bao-nats-callout-xkey";
|
||||
TimeoutStartSec = 30;
|
||||
};
|
||||
environment = {
|
||||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
}
|
||||
// lib.optionalAttrs authStoreActive {
|
||||
BAO_CLIENT_CERT = deployCfg.nats.authBaoClientCertFile;
|
||||
BAO_CLIENT_KEY = deployCfg.nats.authBaoClientKeyFile;
|
||||
}
|
||||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||||
BAO_CACERT = baoDeploy.serverCaFile;
|
||||
};
|
||||
script =
|
||||
if !authStoreActive then
|
||||
''
|
||||
echo "the swarm queue responder has no store identity, so the callout xkey can be neither created nor fetched and the queue does not start: set services.hyperhive.deploy.nats.authBaoClientCertFile and authBaoClientKeyFile" >&2
|
||||
exit 1
|
||||
''
|
||||
else
|
||||
''
|
||||
set -euo pipefail
|
||||
|
||||
${atomicWriteSecret}
|
||||
${refreshConsumer}
|
||||
|
||||
swarm-nats-auth ensure-xkey --store-cert-role ${lib.escapeShellArg authCertRole}
|
||||
|
||||
err="$(mktemp)"
|
||||
trap 'rm -f "$err"' EXIT
|
||||
|
||||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||||
echo "could not log in to swarm-bao with the queue responder's certificate; leaving the callout xkey file as it is." >&2
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
export BAO_TOKEN
|
||||
|
||||
if ! xkey="$(bao kv get -field=value ${lib.escapeShellArg xkeyPublicPath} 2>"$err")"; then
|
||||
echo "swarm-bao did not return ${xkeyPublicPath}." >&2
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$xkey" ]; then
|
||||
echo "swarm-bao returned an empty ${xkeyPublicPath}." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
install -d -m 0755 ${lib.escapeShellArg hostRuntimeDir}
|
||||
if secret_differs ${lib.escapeShellArg "${hostRuntimeDir}/${xkeyPublicFile}"} "$xkey"; then
|
||||
atomic_write_secret 0444 "" ${lib.escapeShellArg "${hostRuntimeDir}/${xkeyPublicFile}"} "$xkey"
|
||||
fi
|
||||
|
||||
# `renderConfig` reads the file as the server starts, and only then.
|
||||
refresh_consumer ${lib.escapeShellArg machine} nats.service ${lib.escapeShellArg "${hostRuntimeDir}/${xkeyPublicFile}"}
|
||||
'';
|
||||
};
|
||||
|
||||
# ⚠️ Minted on the HOST, not in the container, because the responder is
|
||||
# a separate unit that needs the user seed: generating it inside would
|
||||
# trap it there and require a secret-export path back out — the exact
|
||||
|
|
@ -1070,33 +1250,18 @@ in
|
|||
issuer_pub="$(nk -inkey ${lib.escapeShellArg autoIssuerSeed} -pubout)"
|
||||
|
||||
# Public halves only — world-readable on purpose, since the server
|
||||
# publishes them to every client that connects.
|
||||
# publishes them to every client that connects. `renderConfig` puts
|
||||
# them in the server's config. Mode BEFORE the rename: a rename
|
||||
# publishes whatever the file already is.
|
||||
install -d -m 0755 ${lib.escapeShellArg hostRuntimeDir}
|
||||
sed -e "s|@USER_PUBKEY@|$user_pub|g" \
|
||||
-e "s|@ISSUER_PUBKEY@|$issuer_pub|g" \
|
||||
${calloutTemplate} > ${lib.escapeShellArg hostRuntimeDir}/callout.conf.tmp
|
||||
# Mode BEFORE the rename: a rename publishes whatever the file
|
||||
# already is, so setting it afterwards leaves a window where the
|
||||
# live path has the wrong mode. Same trap as the gateway's
|
||||
# atomic-publish path.
|
||||
chmod 0444 ${lib.escapeShellArg hostRuntimeDir}/callout.conf.tmp
|
||||
mv ${lib.escapeShellArg hostRuntimeDir}/callout.conf.tmp \
|
||||
${lib.escapeShellArg hostRuntimeDir}/callout.conf
|
||||
|
||||
# Upstream's rendered settings, as a sibling the wrapper can name
|
||||
# without a leading slash. Refreshed unconditionally — unlike the
|
||||
# seeds, this one MUST track the current system, and a stale copy
|
||||
# would silently run yesterday's config.
|
||||
ln -sfn ${renderedSettings} ${lib.escapeShellArg hostRuntimeDir}/settings.conf
|
||||
|
||||
# The wrapper. Bare filenames — see `runtimeDir`. printf rather than
|
||||
# a heredoc, whose terminator would depend on nix's indentation
|
||||
# stripping and break the next time `nix fmt` touched this block.
|
||||
printf 'include "settings.conf"\ninclude "callout.conf"\n' \
|
||||
> ${lib.escapeShellArg hostRuntimeDir}/nats.conf.tmp
|
||||
chmod 0444 ${lib.escapeShellArg hostRuntimeDir}/nats.conf.tmp
|
||||
mv ${lib.escapeShellArg hostRuntimeDir}/nats.conf.tmp \
|
||||
${lib.escapeShellArg hostRuntimeDir}/nats.conf
|
||||
printf '%s\n' "$user_pub" > ${lib.escapeShellArg "${hostRuntimeDir}/${userPublicFile}"}.tmp
|
||||
printf '%s\n' "$issuer_pub" > ${lib.escapeShellArg "${hostRuntimeDir}/${issuerPublicFile}"}.tmp
|
||||
chmod 0444 ${lib.escapeShellArg "${hostRuntimeDir}/${userPublicFile}"}.tmp \
|
||||
${lib.escapeShellArg "${hostRuntimeDir}/${issuerPublicFile}"}.tmp
|
||||
mv ${lib.escapeShellArg "${hostRuntimeDir}/${userPublicFile}"}.tmp \
|
||||
${lib.escapeShellArg "${hostRuntimeDir}/${userPublicFile}"}
|
||||
mv ${lib.escapeShellArg "${hostRuntimeDir}/${issuerPublicFile}"}.tmp \
|
||||
${lib.escapeShellArg "${hostRuntimeDir}/${issuerPublicFile}"}
|
||||
'';
|
||||
};
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue