swarm-nats: take the callout issuer and user keys from the store
`swarm-bao-nats-callout-xkey` runs `ensure-callout-keys`, importing
/var/lib/swarm-nats-callout/{issuer,callout-user}.seed when present so the
keys keep their public halves, then fetches all three public halves from
the store. The verb removes each seed file once a read of the store returns
a seed; when the store's seed differs, it is kept and a warning names the
file. No tmpfiles rule touches those files, since one could run before the
import with the store sealed and leave the next run to mint a new key.
The server renders the issuer and user public halves from the fetched files,
checked for shape as the xkey's is, and does not start without them. The
responder reads both seeds from the store, so the copies into the container
and their LoadCredential entries are gone, and a tmpfiles rule removes the
copies a container still holds. `swarm-nats-callout-keys` is removed.
On the store's host the fetch unit is also ordered after
`swarm-bao-secret-publisher-policy`, so the publisher's denial of the seed
paths is written before the unit creates or keeps a seed there.
`deploy.nats.autoGenerateCallout`, `calloutUserPublicKey`,
`calloutIssuerPublicKey`, `calloutUserSeedFile` and `calloutIssuerSeedFile`
are removed with `mkRemovedOptionModule`, under `swarm.nats` too, each
message saying the keys live in the store.
This commit is contained in:
atlas
mara
parent
66ac034949
commit
c10cc0e018
12 changed files with 427 additions and 556 deletions
|
|
@ -65,6 +65,7 @@
|
|||
./swarm-wireguard.nix
|
||||
./swarm.nix
|
||||
./host-ca-removed.nix
|
||||
./nats-callout-removed.nix
|
||||
./swarm-peers-removed.nix
|
||||
./swarm-required-services.nix
|
||||
];
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@
|
|||
#
|
||||
# ⚠️ Each entry is an attrset with an `enable`, not a bare bool, so a
|
||||
# service that grows a second *deployment* decision has somewhere to put
|
||||
# it — `deploy.nats = { enable; autoGenerateCallout; }` is then an ordinary
|
||||
# it — `deploy.nats = { enable; authPackage; }` is then an ordinary
|
||||
# addition rather than a migration, where a bare bool leaves it
|
||||
# unrepresentable.
|
||||
#
|
||||
|
|
@ -197,38 +197,6 @@ in
|
|||
[ "services" "hyperhive" "deploy" "ci-runner" "package" ]
|
||||
)
|
||||
|
||||
# The queue's callout identity. What stays under `swarm.nats` is what the
|
||||
# queue IS to every hive — the ports it answers on and the client id it is
|
||||
# registered under; these five are what the host running it decides. The
|
||||
# responder package is one of them, renamed further down with the other
|
||||
# `*.package` entries.
|
||||
#
|
||||
# ⚠️ The two PUBLIC keys follow their seeds. A public key looks
|
||||
# swarm-wide, and the user key genuinely is published to every client that
|
||||
# connects — but peers RECEIVE it over the wire, they never configure it,
|
||||
# and splitting a keypair across two namespaces is worse than either
|
||||
# placement. `enable`'s shim is the one above.
|
||||
(lib.mkRenamedOptionModule
|
||||
[ "services" "hyperhive" "swarm" "nats" "autoGenerateCallout" ]
|
||||
[ "services" "hyperhive" "deploy" "nats" "autoGenerateCallout" ]
|
||||
)
|
||||
(lib.mkRenamedOptionModule
|
||||
[ "services" "hyperhive" "swarm" "nats" "calloutUserPublicKey" ]
|
||||
[ "services" "hyperhive" "deploy" "nats" "calloutUserPublicKey" ]
|
||||
)
|
||||
(lib.mkRenamedOptionModule
|
||||
[ "services" "hyperhive" "swarm" "nats" "calloutIssuerPublicKey" ]
|
||||
[ "services" "hyperhive" "deploy" "nats" "calloutIssuerPublicKey" ]
|
||||
)
|
||||
(lib.mkRenamedOptionModule
|
||||
[ "services" "hyperhive" "swarm" "nats" "calloutUserSeedFile" ]
|
||||
[ "services" "hyperhive" "deploy" "nats" "calloutUserSeedFile" ]
|
||||
)
|
||||
(lib.mkRenamedOptionModule
|
||||
[ "services" "hyperhive" "swarm" "nats" "calloutIssuerSeedFile" ]
|
||||
[ "services" "hyperhive" "deploy" "nats" "calloutIssuerSeedFile" ]
|
||||
)
|
||||
|
||||
# The rest of the forge split. What stays under `swarm.forge` is what the
|
||||
# forge IS from any hive's point of view — the names and ports it answers
|
||||
# on, the URLs it advertises, the client id it is registered under; these
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
# Glue: where the store and one of its readers share a host, the reader waits
|
||||
# for the unit that writes the cert-auth role it logs in with.
|
||||
# for the unit that writes the cert-auth role it logs in with, and for any
|
||||
# other policy unit whose rules it relies on.
|
||||
#
|
||||
# ONE PAIRING PER FILE — the store's reader policy units ← the readers
|
||||
# they grant, and nothing else. Deleting this leaves every reader as it is on a
|
||||
# ONE PAIRING PER FILE — the store's policy units ← the readers that rely
|
||||
# on them, and nothing else. Deleting this leaves every reader as it is on a
|
||||
# host whose store is remote: it may log in before its role exists, and its own
|
||||
# retries are what carry it past that.
|
||||
#
|
||||
|
|
@ -40,7 +41,7 @@ let
|
|||
swarm-bao-otel-oidc =
|
||||
deployCfg.swarm-otel.enable
|
||||
&& havePair baoDeploy.otelOidcClientCertFile baoDeploy.otelOidcClientKeyFile;
|
||||
# ./swarm-bao.nix: its block is gated on the store, which `orderAfterPolicy`
|
||||
# ./swarm-bao.nix: its block is gated on the store, which `orderAfterPolicies`
|
||||
# already checks.
|
||||
swarm-bao-forwarder-oidc = true;
|
||||
# ./swarm-nats.nix: the queue's TLS leaf, not a secret, but the same wait.
|
||||
|
|
@ -50,37 +51,43 @@ let
|
|||
};
|
||||
|
||||
# Readers that log in under another unit's principal: reader unit → the
|
||||
# gate its own module defines it under, and that principal's policy unit.
|
||||
# gate its own module defines it under, and the policy units it waits for.
|
||||
readersOfOtherPolicy = {
|
||||
# ./swarm-nats.nix, under the queue responder's leaf.
|
||||
swarm-bao-nats-auth-oidc = {
|
||||
here = deployCfg.nats.enable;
|
||||
policy = "swarm-bao-nats-auth-policy";
|
||||
policies = [ "swarm-bao-nats-auth-policy" ];
|
||||
};
|
||||
# ./swarm-nats.nix, under the same leaf.
|
||||
# ./swarm-nats.nix, under the same leaf. Also after the publisher's
|
||||
# policy, which denies the publisher the callout seeds' paths: until it
|
||||
# is written the publisher may create them, and this unit keeps a seed it
|
||||
# finds there.
|
||||
swarm-bao-nats-callout-xkey = {
|
||||
here = deployCfg.nats.enable;
|
||||
policy = "swarm-bao-nats-auth-policy";
|
||||
policies = [
|
||||
"swarm-bao-nats-auth-policy"
|
||||
"swarm-bao-secret-publisher-policy"
|
||||
];
|
||||
};
|
||||
# ./hive-matrix.nix, under matrix-ctl's leaf.
|
||||
swarm-bao-matrix-oidc = {
|
||||
here = deployCfg.matrix.enable;
|
||||
policy = "swarm-bao-matrix-ctl-policy";
|
||||
policies = [ "swarm-bao-matrix-ctl-policy" ];
|
||||
};
|
||||
};
|
||||
|
||||
orderAfterPolicy =
|
||||
reader: here: policy:
|
||||
orderAfterPolicies =
|
||||
reader: here: policies:
|
||||
lib.mkIf (baoDeploy.enable && here) {
|
||||
systemd.services.${reader} = {
|
||||
after = [ "${policy}.service" ];
|
||||
wants = [ "${policy}.service" ];
|
||||
after = map (p: "${p}.service") policies;
|
||||
wants = map (p: "${p}.service") policies;
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
config = lib.mkMerge (
|
||||
lib.mapAttrsToList (reader: here: orderAfterPolicy reader here "${reader}-policy") readersHere
|
||||
++ lib.mapAttrsToList (reader: r: orderAfterPolicy reader r.here r.policy) readersOfOtherPolicy
|
||||
lib.mapAttrsToList (reader: here: orderAfterPolicies reader here [ "${reader}-policy" ]) readersHere
|
||||
++ lib.mapAttrsToList (reader: r: orderAfterPolicies reader r.here r.policies) readersOfOtherPolicy
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -74,14 +74,6 @@ in
|
|||
# services" is a per-host decision.
|
||||
config.services.hyperhive.deploy.allSwarmServices = lib.mkDefault cfg.deploy.singleHostSwarm;
|
||||
|
||||
# The queue's auth-callout nkeys. Generating them is safe exactly
|
||||
# when one operator owns both the queue and its responder, which is
|
||||
# what this mode asserts. On any other topology the seeds have to
|
||||
# reach whoever runs the responder, and minting them here would move
|
||||
# that hand-off somewhere less visible rather than removing it.
|
||||
config.services.hyperhive.deploy.nats.autoGenerateCallout =
|
||||
lib.mkDefault cfg.deploy.singleHostSwarm;
|
||||
|
||||
# The controller is asserted by the MODE and by nothing else. Its own
|
||||
# option stays `default = false` precisely because running it is a
|
||||
# statement about swarm topology — but "this box is the whole
|
||||
|
|
|
|||
52
nix/host-modules/nats-callout-removed.nix
Normal file
52
nix/host-modules/nats-callout-removed.nix
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
# Removal shim for the queue's callout nkey options, and for the copies of
|
||||
# their seeds inside the `swarm-nats` container.
|
||||
#
|
||||
# The callout issuer and user keys live in the swarm secret store beside the
|
||||
# callout xkey: `swarm-bao-nats-callout-xkey` in ./swarm-nats.nix creates
|
||||
# them there, and the server renders their public halves from the store. A
|
||||
# definition of a removed option fails evaluation with a message, under its
|
||||
# `deploy.nats` path and under the `swarm.nats` one older configs use.
|
||||
#
|
||||
# The host seed files under /var/lib/swarm-nats-callout are NOT removed here:
|
||||
# that unit stores each one before it removes it, and a tmpfiles rule could
|
||||
# run first.
|
||||
#
|
||||
# Self-contained and deletable, like ./host-ca-removed.nix: one file to
|
||||
# remove when the deprecation window closes.
|
||||
{ lib, ... }:
|
||||
let
|
||||
inStore = ''
|
||||
The queue's callout issuer and user keys live in the swarm secret store:
|
||||
swarm-bao-nats-callout-xkey creates them there, and the server renders
|
||||
their public halves from it. Remove this definition. See
|
||||
docs/swarm/secrets.md.
|
||||
'';
|
||||
# The seed-file options name a key an operator may want to keep.
|
||||
seedFile = name: ''
|
||||
${inStore}
|
||||
A seed at /var/lib/swarm-nats-callout/${name} is what that unit stores
|
||||
when the store holds none, so a key placed there keeps its public half;
|
||||
otherwise a new key is created.
|
||||
'';
|
||||
removed = name: message: [
|
||||
(lib.mkRemovedOptionModule [ "services" "hyperhive" "deploy" "nats" name ] message)
|
||||
(lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "nats" name ] message)
|
||||
];
|
||||
|
||||
containerSeedDir = "/var/lib/nixos-containers/swarm-nats/var/lib/swarm-nats-auth";
|
||||
in
|
||||
{
|
||||
imports = lib.concatLists [
|
||||
(removed "autoGenerateCallout" inStore)
|
||||
(removed "calloutUserPublicKey" inStore)
|
||||
(removed "calloutIssuerPublicKey" inStore)
|
||||
(removed "calloutUserSeedFile" (seedFile "callout-user.seed"))
|
||||
(removed "calloutIssuerSeedFile" (seedFile "issuer.seed"))
|
||||
];
|
||||
|
||||
# The responder reads both seeds from the store and names neither file.
|
||||
systemd.tmpfiles.rules = map (f: "r ${containerSeedDir}/${f}") [
|
||||
"callout-user.seed"
|
||||
"issuer.seed"
|
||||
];
|
||||
}
|
||||
|
|
@ -48,34 +48,17 @@ let
|
|||
secretDir = "/var/lib/nixos-containers/${machine}${secretDirInContainer}";
|
||||
hostPath = name: "${secretDir}/${name}";
|
||||
|
||||
# The responder needs all three credentials. Gating on them rather than
|
||||
# on `deployCfg.nats.enable` keeps a half-configured hive at "queue up, denying
|
||||
# everyone" instead of "unit crash-looping on a missing file".
|
||||
#
|
||||
# In auto mode the seeds are minted on this host before the container
|
||||
# starts, so they are configured by construction.
|
||||
responderConfigured =
|
||||
deployCfg.nats.autoGenerateCallout
|
||||
|| (deployCfg.nats.calloutUserSeedFile != "" && deployCfg.nats.calloutIssuerSeedFile != "");
|
||||
# Where swarm-secret-publish on authelia's host leaves this client's secret,
|
||||
# composed from the same swarm-wide client id it publishes under. The
|
||||
# responder's store role grants read on this one path.
|
||||
storeSecretPath = "secret/swarm/services/${cfg.clientId}/oidc/client";
|
||||
introspectionUrl = "${toString autheliaUrl}/api/oidc/introspection";
|
||||
|
||||
# Where the responder's seeds actually come from. One name for two
|
||||
# origins, so everything downstream stops caring which mode it is in.
|
||||
userSeedFile =
|
||||
if deployCfg.nats.autoGenerateCallout then autoUserSeed else deployCfg.nats.calloutUserSeedFile;
|
||||
issuerSeedFile =
|
||||
if deployCfg.nats.autoGenerateCallout then autoIssuerSeed else deployCfg.nats.calloutIssuerSeedFile;
|
||||
|
||||
# Seeds stay on the host at 0600 and never enter the container or the
|
||||
# store: only the responder needs them, and it reads them by
|
||||
# `LoadCredential` from here.
|
||||
autoSeedDir = "/var/lib/swarm-nats-callout";
|
||||
autoUserSeed = "${autoSeedDir}/callout-user.seed";
|
||||
autoIssuerSeed = "${autoSeedDir}/issuer.seed";
|
||||
# Host seed files `ensure-callout-keys` stores when the secret store holds
|
||||
# no seed for that key, and removes once it holds one. Never written here.
|
||||
importSeedDir = "/var/lib/swarm-nats-callout";
|
||||
importUserSeed = "${importSeedDir}/callout-user.seed";
|
||||
importIssuerSeed = "${importSeedDir}/issuer.seed";
|
||||
|
||||
# The runtime config directory: wrapper, settings symlink and fragment.
|
||||
# World-readable is correct (everything in it is public) and it is not in
|
||||
|
|
@ -102,9 +85,9 @@ let
|
|||
config.containers.${machine}.config.services.nats.settings
|
||||
);
|
||||
|
||||
# Defined once, rendered twice: into `settings` with the operator's values,
|
||||
# and into the runtime fragment with placeholders filled in at server start
|
||||
# (the xkey always, the two nkeys in auto mode).
|
||||
# Defined once, rendered twice: into `settings` with no keys, and into the
|
||||
# runtime fragment with placeholders for the three public keys, filled in
|
||||
# at server start.
|
||||
#
|
||||
# 🪤 Do not inline these and hand-write the fragment instead. Both files are
|
||||
# loaded and the fragment is the LATER definition, so it wins — a
|
||||
|
|
@ -142,9 +125,9 @@ let
|
|||
# hole. So the only way to get a live server here is a real key
|
||||
# whose seed nobody but the responder holds.
|
||||
#
|
||||
# 🔒 That property is what makes auto mode safe: it renders as ""
|
||||
# there, so any field the fragment fails to override keeps a value
|
||||
# the server refuses to start on. An incomplete merge cannot leave
|
||||
# 🔒 That property is what makes `settings` safe: both nkeys render
|
||||
# as "" there, so any field the fragment fails to override keeps a
|
||||
# value the server refuses to start on. An incomplete merge cannot leave
|
||||
# a walk-in-able server.
|
||||
${calloutAccount}.users = [ { nkey = userKey; } ];
|
||||
# ⚠️ `services.nats.jetstream = true` gives the SERVER JetStream;
|
||||
|
|
@ -200,47 +183,58 @@ let
|
|||
};
|
||||
};
|
||||
|
||||
# The callout's three public keys: where the store keeps each, the file
|
||||
# `swarm-bao-nats-callout-xkey` leaves it in, the placeholder it fills in
|
||||
# `calloutTemplate`, and the shape `renderConfig` checks it has. The store
|
||||
# paths are `swarm-nats-auth`'s `xkey::PUBLIC_PATH` and the `public_path` of
|
||||
# its `nkey::ISSUER` and `nkey::USER`, readable by the `swarm-nats-auth` role
|
||||
# alone; each seed beside them is in ./swarm-bao.nix.
|
||||
calloutPublicKeys = [
|
||||
{
|
||||
storePath = "secret/swarm/public/swarm-nats/callout-xkey";
|
||||
file = "callout-xkey.pub";
|
||||
placeholder = "@XKEY_PUBKEY@";
|
||||
var = "xkey";
|
||||
pattern = "X[A-Z2-7]{55}";
|
||||
kind = "public xkey";
|
||||
}
|
||||
{
|
||||
storePath = "secret/swarm/public/swarm-nats/callout-issuer";
|
||||
file = "issuer.pub";
|
||||
placeholder = "@ISSUER_PUBKEY@";
|
||||
var = "issuer_pub";
|
||||
pattern = "A[A-Z2-7]{55}";
|
||||
kind = "public account nkey";
|
||||
}
|
||||
{
|
||||
storePath = "secret/swarm/public/swarm-nats/callout-user";
|
||||
file = "callout-user.pub";
|
||||
placeholder = "@USER_PUBKEY@";
|
||||
var = "user_pub";
|
||||
pattern = "U[A-Z2-7]{55}";
|
||||
kind = "public user nkey";
|
||||
}
|
||||
];
|
||||
|
||||
# The fragment as nix renders it, with placeholders where the runtime
|
||||
# values go. Rendered by the same JSON generator upstream uses, so the
|
||||
# fragment is generated rather than transcribed — NATS' config parser
|
||||
# accepts JSON, and an `include` of it merges (measured).
|
||||
calloutTemplate = natsFormat.generate "swarm-nats-callout-template.conf" (calloutBlocks {
|
||||
userKey =
|
||||
if deployCfg.nats.autoGenerateCallout then "@USER_PUBKEY@" else deployCfg.nats.calloutUserPublicKey;
|
||||
issuerKey =
|
||||
if deployCfg.nats.autoGenerateCallout then
|
||||
"@ISSUER_PUBKEY@"
|
||||
else
|
||||
deployCfg.nats.calloutIssuerPublicKey;
|
||||
userKey = "@USER_PUBKEY@";
|
||||
issuerKey = "@ISSUER_PUBKEY@";
|
||||
xkey = "@XKEY_PUBKEY@";
|
||||
});
|
||||
|
||||
# Where the fetch unit leaves the xkey's public half, read from the store at
|
||||
# `xkeyPublicPath`. In auto mode the nkeys' public halves sit beside it.
|
||||
xkeyPublicFile = "callout-xkey.pub";
|
||||
userPublicFile = "callout-user.pub";
|
||||
issuerPublicFile = "issuer.pub";
|
||||
|
||||
# The callout xkey's public half in the store, readable by the
|
||||
# `swarm-nats-auth` role alone. The Rust half is `swarm-nats-auth`'s
|
||||
# `xkey::PUBLIC_PATH`; the seed beside it is `xkeySeedLeaf` in
|
||||
# ./swarm-bao.nix.
|
||||
xkeyPublicPath = "secret/swarm/public/swarm-nats/callout-xkey";
|
||||
|
||||
# Each placeholder in `calloutTemplate`, as a `sed` expression over the
|
||||
# shell variable `renderConfig` reads its value into.
|
||||
calloutSubstitutions = [
|
||||
''-e "s|@XKEY_PUBKEY@|$xkey|g"''
|
||||
]
|
||||
++ lib.optionals deployCfg.nats.autoGenerateCallout [
|
||||
''-e "s|@USER_PUBKEY@|$user_pub|g"''
|
||||
''-e "s|@ISSUER_PUBKEY@|$issuer_pub|g"''
|
||||
];
|
||||
calloutSubstitutions = map (k: ''-e "s|${k.placeholder}|${"$"}${k.var}|g"'') calloutPublicKeys;
|
||||
|
||||
# Renders the server's config as it starts, so the settings it includes are
|
||||
# always the running system's. Exits non-zero, and so stops the server
|
||||
# starting, while no xkey has been fetched: the server never runs without
|
||||
# one. `+` because the server's own sandbox cannot write `runtimeDir`.
|
||||
# starting, while any of the three public keys has not been fetched: the
|
||||
# server never runs without them. `+` because the server's own sandbox
|
||||
# cannot write `runtimeDir`.
|
||||
renderConfig = pkgs.writeShellScript "swarm-nats-render-config" ''
|
||||
set -euo pipefail
|
||||
PATH=${
|
||||
|
|
@ -252,21 +246,19 @@ let
|
|||
}
|
||||
d=${lib.escapeShellArg runtimeDir}
|
||||
|
||||
if [ ! -s "$d/${xkeyPublicFile}" ]; then
|
||||
echo "${runtimeDir}/${xkeyPublicFile} is missing: swarm-bao-nats-callout-xkey on the host fetches it from the secret store, and the queue does not start without it" >&2
|
||||
exit 1
|
||||
fi
|
||||
xkey="$(cat "$d/${xkeyPublicFile}")"
|
||||
# A public curve key is X and 55 base32 characters; checked here because
|
||||
# it is about to become part of a sed expression.
|
||||
if ! printf '%s' "$xkey" | grep -Eqx 'X[A-Z2-7]{55}'; then
|
||||
echo "${runtimeDir}/${xkeyPublicFile} does not hold a public xkey" >&2
|
||||
exit 1
|
||||
fi
|
||||
${lib.optionalString deployCfg.nats.autoGenerateCallout ''
|
||||
user_pub="$(cat "$d/${userPublicFile}")"
|
||||
issuer_pub="$(cat "$d/${issuerPublicFile}")"
|
||||
''}
|
||||
${lib.concatMapStrings (k: ''
|
||||
if [ ! -s "$d/${k.file}" ]; then
|
||||
echo "${runtimeDir}/${k.file} is missing: swarm-bao-nats-callout-xkey on the host fetches it from the secret store, and the queue does not start without it" >&2
|
||||
exit 1
|
||||
fi
|
||||
${k.var}="$(cat "$d/${k.file}")"
|
||||
# The key type's letter and 55 base32 characters; checked here because
|
||||
# it is about to become part of a sed expression.
|
||||
if ! printf '%s' "${"$"}${k.var}" | grep -Eqx '${k.pattern}'; then
|
||||
echo "${runtimeDir}/${k.file} does not hold a ${k.kind}" >&2
|
||||
exit 1
|
||||
fi
|
||||
'') calloutPublicKeys}
|
||||
sed ${lib.concatStringsSep " " calloutSubstitutions} ${calloutTemplate} > "$d/callout.conf.tmp"
|
||||
# Mode BEFORE the rename: a rename publishes whatever the file already
|
||||
# is. World-readable, because the server reads it as `nats` and
|
||||
|
|
@ -350,14 +342,10 @@ in
|
|||
|
||||
# What the queue IS to every hive, the ports it answers on and the client id
|
||||
# it is registered under, is `swarm.nats` in ./swarm-nats-service.nix. What
|
||||
# the host running it decides is below — which responder build it runs,
|
||||
# whether it mints its own keypairs, and where the seeds sit. `enable`
|
||||
# already lives in ./deploy.nix, which also carries the renames.
|
||||
#
|
||||
# ⚠️ The two PUBLIC keys move with their seeds rather than staying: peers
|
||||
# receive the user key over the wire when they connect (docs/swarm/secrets.md),
|
||||
# they never configure it, and splitting a keypair across two namespaces is
|
||||
# worse than either placement.
|
||||
# the host running it decides is below — which responder build it runs and
|
||||
# the store identities its units present. `enable` already lives in
|
||||
# ./deploy.nix, which also carries the renames. The callout's keys are no
|
||||
# option at all: they live in the secret store (docs/swarm/secrets.md).
|
||||
options.services.hyperhive.deploy.nats = {
|
||||
authPackage = lib.mkOption {
|
||||
type = lib.types.package;
|
||||
|
|
@ -375,110 +363,6 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
autoGenerateCallout = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
example = true;
|
||||
description = ''
|
||||
Generate the auth-callout nkeys on this host instead of taking
|
||||
them from `calloutUserPublicKey` / `calloutIssuerPublicKey`.
|
||||
|
||||
A first-boot unit mints both keypairs if absent, keeps the seeds
|
||||
host-side at `0600`, and writes only the public halves where the
|
||||
server's config is rendered from as it starts. Nothing secret is
|
||||
evaluated, so nothing secret reaches the nix store.
|
||||
|
||||
Leave it off wherever the queue and its clients are not the same
|
||||
operator's problem: the seeds must reach whoever runs the
|
||||
responder, and minting them here only moves that distribution
|
||||
somewhere less visible. `singleHostSwarm` turns it on.
|
||||
'';
|
||||
};
|
||||
|
||||
calloutUserPublicKey = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "";
|
||||
example = "UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCBBQ";
|
||||
description = ''
|
||||
Public half of the **user** nkey the auth-callout responder
|
||||
authenticates as.
|
||||
|
||||
`auth_callout.auth_users` exempts this identity from needing
|
||||
callout approval — it is the one that answers auth requests, so
|
||||
it cannot wait for itself. **That exemption is exactly why it
|
||||
needs a credential of its own**: without one the escape hatch is
|
||||
an open door, and it stands open to everything that can reach the
|
||||
queue — every process on the host serving it, and every remote
|
||||
hive in a multi-host swarm.
|
||||
|
||||
An nkey rather than a password for the same reason
|
||||
`calloutIssuerPublicKey` is: only the public half appears here,
|
||||
and nix renders it into the world-readable store harmlessly. The
|
||||
seed reaches the responder and nothing else, so until the
|
||||
responder exists **nobody can authenticate as this user at all**
|
||||
— which is what makes a hive with no responder genuinely closed
|
||||
rather than merely gated.
|
||||
|
||||
Required when `deploy.nats.enable` is set.
|
||||
'';
|
||||
};
|
||||
|
||||
calloutIssuerPublicKey = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "";
|
||||
example = "ACYR44YO3XZRZBJIYLI5SL6LOPIW37JTD52LNOBHUE34XMH7N5ABMFJH";
|
||||
description = ''
|
||||
Public half of the account nkey whose signature the server
|
||||
accepts on a user JWT minted by the auth-callout responder.
|
||||
|
||||
**A public key, and therefore a value rather than a path** —
|
||||
the deliberate exception to the rule that credentials are
|
||||
`*File` options. It is published to every client that connects
|
||||
and its whole job is to be widely known; the matching *seed* is
|
||||
the secret, is never named here, and reaches only the responder.
|
||||
|
||||
Required when `deploy.nats.enable` is set. Without it the server
|
||||
has no issuer to trust and no client can be authorized — which is
|
||||
the fail-closed state described below, but arrived at by accident
|
||||
rather than on purpose, so it fails at eval instead.
|
||||
'';
|
||||
};
|
||||
|
||||
calloutUserSeedFile = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "";
|
||||
example = "/run/secrets/swarm-nats-callout-user.seed";
|
||||
description = ''
|
||||
Absolute host path to the **seed** whose public half is
|
||||
`calloutUserPublicKey`. The responder authenticates to the queue
|
||||
with it.
|
||||
|
||||
A `str` rather than a `path`, and the reason is not style: a
|
||||
`path`-typed literal is hash-copied into the world-readable nix
|
||||
store at eval time, which is the opposite of what a seed wants.
|
||||
Same discipline as `otel.headersCredential`.
|
||||
|
||||
Until this is set the responder cannot start, and the queue
|
||||
stays in its fail-closed state — which is the correct behaviour,
|
||||
not a gap.
|
||||
'';
|
||||
};
|
||||
|
||||
calloutIssuerSeedFile = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "";
|
||||
example = "/run/secrets/swarm-nats-issuer.seed";
|
||||
description = ''
|
||||
Absolute host path to the **account** seed whose public half is
|
||||
`calloutIssuerPublicKey`. The responder signs the user JWTs it
|
||||
issues with it, so possession of this file is the authority to
|
||||
admit anyone to the queue.
|
||||
|
||||
A `str` for the same store-leak reason as
|
||||
`calloutUserSeedFile`.
|
||||
'';
|
||||
};
|
||||
|
||||
baoClientCertFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
|
|
@ -560,73 +444,6 @@ in
|
|||
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
||||
|
||||
assertions = [
|
||||
{
|
||||
# Fail at EVAL, not at boot: a queue that comes up unable to
|
||||
# authenticate anyone presents as every client hanging, which is
|
||||
# several layers from "the operator never set the issuer".
|
||||
assertion = deployCfg.nats.autoGenerateCallout || deployCfg.nats.calloutIssuerPublicKey != "";
|
||||
message = ''
|
||||
services.hyperhive.deploy.nats.enable requires
|
||||
services.hyperhive.deploy.nats.calloutIssuerPublicKey — the
|
||||
public half of the account nkey that signs user JWTs for this
|
||||
queue.
|
||||
|
||||
It is public and belongs in config; the matching seed is a
|
||||
secret and is delivered to the callout responder instead. See
|
||||
docs/swarm/secrets.md for which is which.
|
||||
'';
|
||||
}
|
||||
{
|
||||
# Without this the callout-exempt user has no credential, and
|
||||
# NATS accepts `CONNECT {"user":"auth"}` from anyone. An eval
|
||||
# failure is the only place to catch that: the rendered config is
|
||||
# valid, the server starts, and the hole is invisible until
|
||||
# somebody connects.
|
||||
assertion = deployCfg.nats.autoGenerateCallout || deployCfg.nats.calloutUserPublicKey != "";
|
||||
message = ''
|
||||
services.hyperhive.deploy.nats.enable requires
|
||||
services.hyperhive.deploy.nats.calloutUserPublicKey — the
|
||||
public half of the user nkey the auth-callout responder
|
||||
authenticates as.
|
||||
|
||||
It is exempt from callout approval by design, which is exactly
|
||||
why it needs its own credential: a `users` entry with a name
|
||||
and no key authenticates anyone who sends that name.
|
||||
'';
|
||||
}
|
||||
{
|
||||
# The two above guard the halves the SERVER needs; the responder
|
||||
# needs the other halves, and nothing related them. Satisfying
|
||||
# only the public ones renders a valid `auth_callout` block and
|
||||
# defines no responder — and callout with no responder is the
|
||||
# fail-closed state, so the queue refuses everyone.
|
||||
#
|
||||
# Nothing downstream catches it. `nats-server -t` runs in exactly
|
||||
# this case and passes, because the config IS valid; the missing
|
||||
# piece is a unit, and the absence of a unit is not an event. The
|
||||
# symptom is every client timing out, since a NATS denial reaches
|
||||
# the client as a timeout rather than an error.
|
||||
# The binding itself, not a copy of its formula: what is being
|
||||
# asserted IS "the responder is configured", and two copies of one
|
||||
# boolean is two places for a future edit to land in only one.
|
||||
assertion = responderConfigured;
|
||||
message = ''
|
||||
services.hyperhive.deploy.nats has callout public keys but no
|
||||
seed files:
|
||||
|
||||
deploy.nats.calloutUserSeedFile = "${deployCfg.nats.calloutUserSeedFile}"
|
||||
deploy.nats.calloutIssuerSeedFile = "${deployCfg.nats.calloutIssuerSeedFile}"
|
||||
|
||||
Each seed is the private half of the public key already set
|
||||
here — the server verifies with the public half, the responder
|
||||
signs with the private one. Configuring one side alone leaves
|
||||
this queue with an auth-callout nobody answers, which refuses
|
||||
every client rather than degrading.
|
||||
|
||||
Set both seed paths, or set deploy.nats.autoGenerateCallout = true
|
||||
to have this host mint all four.
|
||||
'';
|
||||
}
|
||||
{
|
||||
# The callout xkey and the responder's OIDC client secret are read
|
||||
# from the store under this identity; without either the server or
|
||||
|
|
@ -754,19 +571,18 @@ in
|
|||
|
||||
serverName = "swarm-nats";
|
||||
port = cfg.port;
|
||||
# Off in every mode now. The `tls` block below names a leaf that
|
||||
# exists only at runtime, and `nats-server -t` loads it, so the
|
||||
# build-time check fails on every hive. In auto mode it failed
|
||||
# already, on callout keys that are empty until the generator
|
||||
# runs. Upstream's own description names the case: disable it
|
||||
# when the config includes other files. The check moves to
|
||||
# Off. The `tls` block below names a leaf that exists only at
|
||||
# runtime, and `nats-server -t` loads it, so the build-time check
|
||||
# fails on every hive; so do the callout keys, empty in
|
||||
# `settings`. Upstream's own description names the case: disable
|
||||
# it when the config includes other files. The check moves to
|
||||
# server start.
|
||||
validateConfig = false;
|
||||
|
||||
settings =
|
||||
calloutBlocks {
|
||||
userKey = deployCfg.nats.calloutUserPublicKey;
|
||||
issuerKey = deployCfg.nats.calloutIssuerPublicKey;
|
||||
userKey = "";
|
||||
issuerKey = "";
|
||||
}
|
||||
// {
|
||||
# The most a single publish may be, in bytes, before the
|
||||
|
|
@ -863,17 +679,11 @@ in
|
|||
# The auth-callout responder: the half that lets the server
|
||||
# above say *yes*. Without it the `auth_callout` block is a
|
||||
# door nobody can open, which is the deliberate interim state.
|
||||
#
|
||||
# ⚠️ It is gated on the seeds being configured rather than on
|
||||
# `deployCfg.nats.enable`, so a half-configured hive gets a running,
|
||||
# refusing queue instead of a unit that crash-loops on a
|
||||
# missing file. A queue that denies everyone is a legible
|
||||
# failure; a restart loop is not.
|
||||
systemd.services.swarm-nats-auth = lib.mkIf responderConfigured {
|
||||
systemd.services.swarm-nats-auth = {
|
||||
description = "swarm queue auth-callout responder";
|
||||
after = [ "nats.service" ];
|
||||
requires = [ "nats.service" ];
|
||||
# `nats.service` too: a server started late, once its xkey arrived,
|
||||
# `nats.service` too: a server started late, once its keys arrived,
|
||||
# brings the responder up with it.
|
||||
wantedBy = [
|
||||
"multi-user.target"
|
||||
|
|
@ -887,8 +697,6 @@ in
|
|||
# container's resolver goes through the bridge, where dnsmasq
|
||||
# answers it.
|
||||
"--nats-url tls://${cfg.domain}:${toString cfg.port}"
|
||||
"--user-seed-file \${CREDENTIALS_DIRECTORY}/callout-user.seed"
|
||||
"--issuer-seed-file \${CREDENTIALS_DIRECTORY}/issuer.seed"
|
||||
"--client-secret-file \${CREDENTIALS_DIRECTORY}/oidc-client.secret"
|
||||
"--client-id ${lib.escapeShellArg cfg.clientId}"
|
||||
# The account admitted clients land in, by NAME: in
|
||||
|
|
@ -945,8 +753,6 @@ in
|
|||
# world-readable via /proc/<pid>/cmdline, so a value there
|
||||
# would be readable by every process on the host netns.
|
||||
LoadCredential = [
|
||||
"callout-user.seed:${inContainer "callout-user.seed"}"
|
||||
"issuer.seed:${inContainer "issuer.seed"}"
|
||||
"oidc-client.secret:${inContainer "oidc-client.secret"}"
|
||||
]
|
||||
++ lib.mapAttrsToList (id: _: "${id}:${inContainer id}") authStoreFiles;
|
||||
|
|
@ -955,7 +761,7 @@ in
|
|||
RestartSec = "5s";
|
||||
SyslogIdentifier = "swarm-nats-auth";
|
||||
};
|
||||
# The store the responder reads the callout xkey and agent
|
||||
# The store the responder reads the callout keys and agent
|
||||
# credentials from.
|
||||
environment = {
|
||||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
|
|
@ -980,25 +786,18 @@ in
|
|||
# container start outright rather than degrading.
|
||||
systemd.services."container@${machine}" = caTrust.containerOrdering;
|
||||
|
||||
# Deliver the responder's seeds and store identity into the container
|
||||
# before it starts. A copy rather than a `bindMounts` entry:
|
||||
# nixos-container refuses to start when a bind source is missing, so one
|
||||
# absent seed would take down the **whole container including the queue**,
|
||||
# not merely the responder.
|
||||
systemd.services.swarm-nats-auth-secrets = lib.mkIf responderConfigured {
|
||||
# Deliver the responder's store identity into the container before it
|
||||
# starts. A copy rather than a `bindMounts` entry: nixos-container refuses
|
||||
# to start when a bind source is missing, so one absent file would take
|
||||
# down the **whole container including the queue**, not merely the
|
||||
# responder.
|
||||
systemd.services.swarm-nats-auth-secrets = {
|
||||
description = "deliver the swarm queue responder's credentials";
|
||||
before = [ "container@swarm-nats.service" ];
|
||||
wantedBy = [ "container@swarm-nats.service" ];
|
||||
# In auto mode the seeds this copies do not exist until the generator
|
||||
# has run. `requires` as well as `after`: if minting fails there is
|
||||
# nothing to deliver, and a copy that silently succeeds with a stale
|
||||
# or absent seed is worse than not running.
|
||||
after =
|
||||
lib.optional deployCfg.nats.autoGenerateCallout "swarm-nats-callout-keys.service"
|
||||
# Where the store's PKI is minted on this host, the responder's leaf is
|
||||
# one of its files. Ordering only: elsewhere the unit does not exist.
|
||||
++ [ "swarm-bao-pki.service" ];
|
||||
requires = lib.optional deployCfg.nats.autoGenerateCallout "swarm-nats-callout-keys.service";
|
||||
# Where the store's PKI is minted on this host, the responder's leaf is
|
||||
# one of its files. Ordering only: elsewhere the unit does not exist.
|
||||
after = [ "swarm-bao-pki.service" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
|
|
@ -1008,10 +807,6 @@ in
|
|||
script = ''
|
||||
set -euo pipefail
|
||||
install -d -m 0700 ${lib.escapeShellArg secretDir}
|
||||
install -m 0400 ${lib.escapeShellArg userSeedFile} \
|
||||
${lib.escapeShellArg (hostPath "callout-user.seed")}
|
||||
install -m 0400 ${lib.escapeShellArg issuerSeedFile} \
|
||||
${lib.escapeShellArg (hostPath "issuer.seed")}
|
||||
''
|
||||
+ lib.concatStrings (
|
||||
lib.mapAttrsToList (id: source: ''
|
||||
|
|
@ -1122,19 +917,25 @@ in
|
|||
'';
|
||||
};
|
||||
|
||||
# The callout xkey's public half, from the store into the file
|
||||
# `renderConfig` reads at every server start. `ensure-xkey` first creates
|
||||
# the pair when the store holds none; it needs no queue, so on a first boot
|
||||
# the key exists before the server that requires it, and nothing here
|
||||
# waits on the responder, which itself waits on the server.
|
||||
# The callout's three public keys, from the store into the files
|
||||
# `renderConfig` reads at every server start. `ensure-callout-keys` first
|
||||
# creates each pair the store holds none of; it needs no queue, so on a
|
||||
# first boot the keys exist before the server that requires them, and
|
||||
# nothing here waits on the responder, which itself waits on the server.
|
||||
#
|
||||
# An issuer or user seed file left on this host is what `ensure-callout-keys`
|
||||
# stores for that key when the store holds none, so the key keeps its
|
||||
# public half. It removes the file once a read of the store returns a seed,
|
||||
# in this run and never before: a removal that ran ahead of the import,
|
||||
# with the store sealed, would leave the next run to mint a new key.
|
||||
#
|
||||
# Before the container and wanted by it, the shape `swarm-bao-nats-tls`
|
||||
# has: a store that comes up later fails this run, the server refuses to
|
||||
# start without the file (or starts with the one last fetched), and the
|
||||
# retry that lands it starts or restarts the server. Under the responder's
|
||||
# own leaf, the one principal that may create the two xkey paths.
|
||||
# start without the files (or starts with the ones last fetched), and the
|
||||
# retry that lands them starts or restarts the server. Under the
|
||||
# responder's own leaf, the one principal that may create the keys' paths.
|
||||
systemd.services.swarm-bao-nats-callout-xkey = {
|
||||
description = "create the swarm queue's callout xkey if absent and fetch its public half for the server";
|
||||
description = "create the swarm queue's callout keys if absent and fetch their public halves for the server";
|
||||
# As `swarm-bao-nats-auth-oidc`: the store's units exist only where it
|
||||
# runs.
|
||||
after = lib.optionals baoDeploy.enable [
|
||||
|
|
@ -1175,91 +976,49 @@ in
|
|||
${atomicWriteSecret}
|
||||
${refreshConsumer}
|
||||
|
||||
swarm-nats-auth ensure-xkey --store-cert-role ${lib.escapeShellArg authCertRole}
|
||||
swarm-nats-auth ensure-callout-keys --store-cert-role ${lib.escapeShellArg authCertRole} \
|
||||
--import-issuer-seed-file ${lib.escapeShellArg importIssuerSeed} \
|
||||
--import-user-seed-file ${lib.escapeShellArg importUserSeed}
|
||||
if [ -d ${lib.escapeShellArg importSeedDir} ]; then
|
||||
rmdir --ignore-fail-on-non-empty ${lib.escapeShellArg importSeedDir}
|
||||
fi
|
||||
|
||||
err="$(mktemp)"
|
||||
trap 'rm -f "$err"' EXIT
|
||||
|
||||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||||
echo "could not log in to swarm-bao with the queue responder's certificate; leaving the callout xkey file as it is." >&2
|
||||
echo "could not log in to swarm-bao with the queue responder's certificate; leaving the callout key files as they are." >&2
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
export BAO_TOKEN
|
||||
|
||||
if ! xkey="$(bao kv get -field=value ${lib.escapeShellArg xkeyPublicPath} 2>"$err")"; then
|
||||
echo "swarm-bao did not return ${xkeyPublicPath}." >&2
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$xkey" ]; then
|
||||
echo "swarm-bao returned an empty ${xkeyPublicPath}." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
install -d -m 0755 ${lib.escapeShellArg hostRuntimeDir}
|
||||
if secret_differs ${lib.escapeShellArg "${hostRuntimeDir}/${xkeyPublicFile}"} "$xkey"; then
|
||||
atomic_write_secret 0444 "" ${lib.escapeShellArg "${hostRuntimeDir}/${xkeyPublicFile}"} "$xkey"
|
||||
fi
|
||||
|
||||
# `renderConfig` reads the file as the server starts, and only then.
|
||||
refresh_consumer ${lib.escapeShellArg machine} nats.service ${lib.escapeShellArg "${hostRuntimeDir}/${xkeyPublicFile}"}
|
||||
'';
|
||||
};
|
||||
|
||||
# ⚠️ Minted on the HOST, not in the container, because the responder is
|
||||
# a separate unit that needs the user seed: generating it inside would
|
||||
# trap it there and require a secret-export path back out — the exact
|
||||
# mechanism this is meant to avoid inventing. Only public halves cross.
|
||||
systemd.services.swarm-nats-callout-keys = lib.mkIf deployCfg.nats.autoGenerateCallout {
|
||||
description = "mint the swarm queue's auth-callout nkeys";
|
||||
before = [ "container@swarm-nats.service" ];
|
||||
wantedBy = [ "container@swarm-nats.service" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
SyslogIdentifier = "swarm-nats-callout-keys";
|
||||
};
|
||||
path = [
|
||||
pkgs.coreutils
|
||||
pkgs.nkeys
|
||||
];
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
install -d -m 0700 ${lib.escapeShellArg autoSeedDir}
|
||||
|
||||
# Mint iff absent. Idempotence is the whole contract: this runs on
|
||||
# every boot, and regenerating would silently invalidate every
|
||||
# credential the responder has already issued against the old
|
||||
# issuer.
|
||||
if [ ! -s ${lib.escapeShellArg autoUserSeed} ]; then
|
||||
nk -gen user > ${lib.escapeShellArg autoUserSeed}.tmp
|
||||
mv ${lib.escapeShellArg autoUserSeed}.tmp ${lib.escapeShellArg autoUserSeed}
|
||||
fi
|
||||
if [ ! -s ${lib.escapeShellArg autoIssuerSeed} ]; then
|
||||
nk -gen account > ${lib.escapeShellArg autoIssuerSeed}.tmp
|
||||
mv ${lib.escapeShellArg autoIssuerSeed}.tmp ${lib.escapeShellArg autoIssuerSeed}
|
||||
fi
|
||||
chmod 0600 ${lib.escapeShellArg autoUserSeed} ${lib.escapeShellArg autoIssuerSeed}
|
||||
|
||||
user_pub="$(nk -inkey ${lib.escapeShellArg autoUserSeed} -pubout)"
|
||||
issuer_pub="$(nk -inkey ${lib.escapeShellArg autoIssuerSeed} -pubout)"
|
||||
|
||||
# Public halves only — world-readable on purpose, since the server
|
||||
# publishes them to every client that connects. `renderConfig` puts
|
||||
# them in the server's config. Mode BEFORE the rename: a rename
|
||||
# publishes whatever the file already is.
|
||||
install -d -m 0755 ${lib.escapeShellArg hostRuntimeDir}
|
||||
printf '%s\n' "$user_pub" > ${lib.escapeShellArg "${hostRuntimeDir}/${userPublicFile}"}.tmp
|
||||
printf '%s\n' "$issuer_pub" > ${lib.escapeShellArg "${hostRuntimeDir}/${issuerPublicFile}"}.tmp
|
||||
chmod 0444 ${lib.escapeShellArg "${hostRuntimeDir}/${userPublicFile}"}.tmp \
|
||||
${lib.escapeShellArg "${hostRuntimeDir}/${issuerPublicFile}"}.tmp
|
||||
mv ${lib.escapeShellArg "${hostRuntimeDir}/${userPublicFile}"}.tmp \
|
||||
${lib.escapeShellArg "${hostRuntimeDir}/${userPublicFile}"}
|
||||
mv ${lib.escapeShellArg "${hostRuntimeDir}/${issuerPublicFile}"}.tmp \
|
||||
${lib.escapeShellArg "${hostRuntimeDir}/${issuerPublicFile}"}
|
||||
'';
|
||||
''
|
||||
+ lib.concatMapStrings (
|
||||
k:
|
||||
let
|
||||
file = "${hostRuntimeDir}/${k.file}";
|
||||
in
|
||||
''
|
||||
if ! key="$(bao kv get -field=value ${lib.escapeShellArg k.storePath} 2>"$err")"; then
|
||||
echo "swarm-bao did not return ${k.storePath}." >&2
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$key" ]; then
|
||||
echo "swarm-bao returned an empty ${k.storePath}." >&2
|
||||
exit 1
|
||||
fi
|
||||
if secret_differs ${lib.escapeShellArg file} "$key"; then
|
||||
atomic_write_secret 0444 "" ${lib.escapeShellArg file} "$key"
|
||||
fi
|
||||
''
|
||||
) calloutPublicKeys
|
||||
# `renderConfig` reads the files as the server starts, and only then.
|
||||
+ lib.concatMapStrings (k: ''
|
||||
refresh_consumer ${lib.escapeShellArg machine} nats.service ${lib.escapeShellArg "${hostRuntimeDir}/${k.file}"}
|
||||
'') calloutPublicKeys;
|
||||
};
|
||||
|
||||
# The queue's TLS leaf, issued by the secret store's `pki/roles/swarm-nats`
|
||||
|
|
|
|||
|
|
@ -21,20 +21,11 @@ let
|
|||
runGroup
|
||||
;
|
||||
|
||||
# The queue's callout identity, fourth split slice. `autoGenerateCallout` is
|
||||
# left FALSE on purpose: that is what makes the seed paths the thing deciding
|
||||
# `responderConfigured`, so the assertion below is about the seeds rather
|
||||
# than about the auto-mint branch. Every one of the seven old paths is
|
||||
# defined — `enable` included, which is why it is spelled the old way here
|
||||
# while the fixture below uses the new one — so dropping any single nats
|
||||
# shim fails the eval, not just the arms read.
|
||||
# The queue's two old `swarm.nats` paths that still rename, `enable` and
|
||||
# `authPackage`, so dropping either shim fails the eval. No store identity,
|
||||
# which the first case below relies on.
|
||||
natsOldPath = hive {
|
||||
swarm.nats.enable = true;
|
||||
swarm.nats.autoGenerateCallout = false;
|
||||
swarm.nats.calloutUserPublicKey = "UTESTUSERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
|
||||
swarm.nats.calloutIssuerPublicKey = "ATESTISSUERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
|
||||
swarm.nats.calloutUserSeedFile = "/run/secrets/nats-user.seed";
|
||||
swarm.nats.calloutIssuerSeedFile = "/run/secrets/nats-issuer.seed";
|
||||
swarm.nats.authPackage = pkgs.emptyDirectory;
|
||||
};
|
||||
|
||||
|
|
@ -53,22 +44,12 @@ let
|
|||
swarm.authelia.usersFile = "/var/lib/test-authelia/users.yml";
|
||||
swarm.authelia.package = pkgs.emptyDirectory;
|
||||
swarm.authelia.bridgePackage = pkgs.emptyDirectory;
|
||||
swarm.nats.autoGenerateCallout = false;
|
||||
swarm.nats.calloutUserPublicKey = "UTESTUSERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
|
||||
swarm.nats.calloutIssuerPublicKey = "ATESTISSUERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
|
||||
swarm.nats.calloutUserSeedFile = "/run/secrets/nats-user.seed";
|
||||
swarm.nats.calloutIssuerSeedFile = "/run/secrets/nats-issuer.seed";
|
||||
};
|
||||
|
||||
# The queue with a store identity for its responder, placed by hand: the
|
||||
# queue host that is not the store's.
|
||||
natsWithStoreIdentity = hive {
|
||||
deploy.nats.enable = true;
|
||||
deploy.nats.autoGenerateCallout = false;
|
||||
deploy.nats.calloutUserPublicKey = "UTESTUSERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
|
||||
deploy.nats.calloutIssuerPublicKey = "ATESTISSUERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
|
||||
deploy.nats.calloutUserSeedFile = "/run/secrets/nats-user.seed";
|
||||
deploy.nats.calloutIssuerSeedFile = "/run/secrets/nats-issuer.seed";
|
||||
deploy.nats.authPackage = pkgs.emptyDirectory;
|
||||
deploy.nats.authBaoClientCertFile = "/etc/pki/nats-auth.pem";
|
||||
deploy.nats.authBaoClientKeyFile = "/etc/pki/nats-auth-key.pem";
|
||||
|
|
@ -92,11 +73,6 @@ let
|
|||
# store here, and no responder leaf named.
|
||||
natsWithHiveReaderOnly = hive {
|
||||
deploy.nats.enable = true;
|
||||
deploy.nats.autoGenerateCallout = false;
|
||||
deploy.nats.calloutUserPublicKey = "UTESTUSERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
|
||||
deploy.nats.calloutIssuerPublicKey = "ATESTISSUERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
|
||||
deploy.nats.calloutUserSeedFile = "/run/secrets/nats-user.seed";
|
||||
deploy.nats.calloutIssuerSeedFile = "/run/secrets/nats-issuer.seed";
|
||||
deploy.nats.authPackage = pkgs.emptyDirectory;
|
||||
deploy.bao.clientCertFile = "/etc/pki/h1.pem";
|
||||
deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem";
|
||||
|
|
@ -205,42 +181,80 @@ let
|
|||
ok = lib.elem "nats.service" (responderOf natsOldPath).wantedBy;
|
||||
}
|
||||
{
|
||||
# The key is created before anything needs it, by the responder's own
|
||||
# verb under its own leaf, then read back from the public path and left
|
||||
# where the server's render reads it. Before the container and wanted by
|
||||
# it, and a fetch that finds nothing fails so `storeRetry` retries.
|
||||
name = "the callout xkey is created, then fetched from the store before the queue's container starts";
|
||||
# The keys are created before anything needs them, by the responder's
|
||||
# own verb under its own leaf, then each public half is read back from
|
||||
# the store and left where the server's render reads it. Before the
|
||||
# container and wanted by it, and a fetch that finds nothing fails so
|
||||
# `storeRetry` retries.
|
||||
name = "the callout keys are created, then their public halves fetched from the store before the queue's container starts";
|
||||
ok =
|
||||
let
|
||||
u = natsWithStoreIdentity.systemd.services.swarm-bao-nats-callout-xkey;
|
||||
s = builtins.unsafeDiscardStringContext u.script;
|
||||
at = needle: lib.stringLength (lib.head (lib.splitString needle s));
|
||||
ensure = "swarm-nats-auth ensure-xkey --store-cert-role swarm-nats-auth";
|
||||
fetch = "bao kv get -field=value secret/swarm/public/swarm-nats/callout-xkey";
|
||||
file = "/var/lib/nixos-containers/swarm-nats/var/lib/nats-callout/callout-xkey.pub";
|
||||
ensure = "swarm-nats-auth ensure-callout-keys --store-cert-role swarm-nats-auth";
|
||||
fetched =
|
||||
leaf: file:
|
||||
let
|
||||
fetch = "bao kv get -field=value secret/swarm/public/swarm-nats/${leaf}";
|
||||
path = "/var/lib/nixos-containers/swarm-nats/var/lib/nats-callout/${file}";
|
||||
in
|
||||
lib.hasInfix fetch s
|
||||
&& at ensure < at fetch
|
||||
&& lib.hasInfix "atomic_write_secret 0444 \"\" ${path}" s
|
||||
&& lib.hasInfix "refresh_consumer swarm-nats nats.service ${path}" s;
|
||||
in
|
||||
lib.elem "container@swarm-nats.service" u.before
|
||||
&& lib.elem "container@swarm-nats.service" u.wantedBy
|
||||
&& u.environment.BAO_CLIENT_CERT == "/etc/pki/nats-auth.pem"
|
||||
&& lib.hasInfix ensure s
|
||||
&& lib.hasInfix fetch s
|
||||
&& at ensure < at fetch
|
||||
&& lib.hasInfix "atomic_write_secret 0444 \"\" ${file}" s
|
||||
&& lib.hasInfix "refresh_consumer swarm-nats nats.service ${file}" s
|
||||
&& fetched "callout-xkey" "callout-xkey.pub"
|
||||
&& fetched "callout-issuer" "issuer.pub"
|
||||
&& fetched "callout-user" "callout-user.pub"
|
||||
&& !(lib.hasInfix "exit 0" s)
|
||||
&& u.serviceConfig.Restart == "on-failure";
|
||||
}
|
||||
{
|
||||
# Its creates are what the responder's role was granted; on the store's
|
||||
# host the role's unit has to have run first.
|
||||
name = "on the store's host the callout xkey fetch waits for the responder's role";
|
||||
# The seeds a host kept on disk are what the verb stores when the store
|
||||
# holds none, so the keys keep their public halves; the verb removes
|
||||
# them itself. Nothing mints a seed on the host, and no
|
||||
# tmpfiles rule touches the host seeds, which could run before the
|
||||
# import does.
|
||||
name = "the callout keys import the host's seed files, which only the importing unit removes";
|
||||
ok =
|
||||
let
|
||||
s = natsWithStoreIdentity.systemd.services.swarm-bao-nats-callout-xkey.script;
|
||||
rules = natsWithStoreIdentity.systemd.tmpfiles.rules;
|
||||
in
|
||||
lib.hasInfix "--import-issuer-seed-file /var/lib/swarm-nats-callout/issuer.seed" s
|
||||
&& lib.hasInfix "--import-user-seed-file /var/lib/swarm-nats-callout/callout-user.seed" s
|
||||
&& !(natsWithStoreIdentity.systemd.services ? swarm-nats-callout-keys)
|
||||
&& !(lib.any (lib.hasInfix "/var/lib/swarm-nats-callout") rules);
|
||||
}
|
||||
{
|
||||
# Its creates are what the responder's role was granted, and the
|
||||
# publisher's policy is what denies the publisher the seeds' paths: a
|
||||
# seed found there is kept. On the store's host both units run first.
|
||||
name = "on the store's host the callout key fetch waits for the responder's role and the publisher's denial";
|
||||
ok =
|
||||
let
|
||||
u = natsOnStoreHost.systemd.services.swarm-bao-nats-callout-xkey;
|
||||
waitsFor = p: lib.elem "${p}.service" u.after && lib.elem "${p}.service" u.wants;
|
||||
in
|
||||
u.environment.BAO_CLIENT_CERT == "/var/lib/swarm-bao-pki/nats-auth.pem"
|
||||
&& lib.elem "swarm-bao-nats-auth-policy.service" u.after
|
||||
&& lib.elem "swarm-bao-nats-auth-policy.service" u.wants;
|
||||
&& waitsFor "swarm-bao-nats-auth-policy"
|
||||
&& waitsFor "swarm-bao-secret-publisher-policy"
|
||||
&& natsOnStoreHost.systemd.services ? swarm-bao-secret-publisher-policy;
|
||||
}
|
||||
{
|
||||
# The control is the fixture above the store: off the store's host
|
||||
# there is no local policy unit to wait for.
|
||||
name = "off the store's host the callout key fetch waits for no policy unit";
|
||||
ok =
|
||||
let
|
||||
u = natsWithStoreIdentity.systemd.services.swarm-bao-nats-callout-xkey;
|
||||
in
|
||||
!(lib.any (lib.hasSuffix "-policy.service") (u.after ++ u.wants));
|
||||
}
|
||||
{
|
||||
# The responder refuses the hive's shared agent client and takes no flag
|
||||
|
|
@ -318,9 +332,10 @@ let
|
|||
&& lib.hasPrefix "https://" u.environment.BAO_ADDR;
|
||||
}
|
||||
{
|
||||
# Copied as the seeds are, so an absent leaf fails the unit rather than
|
||||
# reaching the responder as a file it cannot log in with.
|
||||
name = "the copy unit delivers the store identity beside the two seeds, and nothing in its place when absent";
|
||||
# Copied so an absent leaf fails the unit rather than reaching the
|
||||
# responder as a file it cannot log in with. No seed is copied: the
|
||||
# responder reads every callout seed from the store.
|
||||
name = "the copy unit delivers the store identity and no seed, and nothing in its place when absent";
|
||||
ok =
|
||||
let
|
||||
s = natsWithStoreIdentity.systemd.services.swarm-nats-auth-secrets.script;
|
||||
|
|
@ -328,8 +343,51 @@ let
|
|||
lib.hasInfix "install -m 0400 /etc/pki/nats-auth.pem " s
|
||||
&& lib.hasInfix "install -m 0400 /etc/pki/nats-auth-key.pem " s
|
||||
&& !(lib.hasInfix "/dev/null" s)
|
||||
&& lib.hasInfix "callout-user.seed" s
|
||||
&& lib.hasInfix "issuer.seed" s;
|
||||
&& !(lib.hasInfix ".seed" s);
|
||||
}
|
||||
{
|
||||
# The responder names no seed file, by flag or by credential, and the
|
||||
# copies a container may still hold are removed. The OIDC secret is the
|
||||
# control that the credential list is read at all.
|
||||
name = "the responder takes no seed from a file, and the container's seed copies are removed";
|
||||
ok =
|
||||
let
|
||||
u = responderOf natsWithStoreIdentity;
|
||||
creds = u.serviceConfig.LoadCredential;
|
||||
dir = "/var/lib/nixos-containers/swarm-nats/var/lib/swarm-nats-auth";
|
||||
rules = natsWithStoreIdentity.systemd.tmpfiles.rules;
|
||||
in
|
||||
lib.elem "oidc-client.secret:/var/lib/swarm-nats-auth/oidc-client.secret" creds
|
||||
&& !(lib.any (lib.hasInfix ".seed") creds)
|
||||
&& !(lib.hasInfix "seed-file" u.serviceConfig.ExecStart)
|
||||
&& lib.elem "r ${dir}/issuer.seed" rules
|
||||
&& lib.elem "r ${dir}/callout-user.seed" rules;
|
||||
}
|
||||
{
|
||||
# Under either namespace, each removed option fails evaluation naming
|
||||
# itself and the store. The control is a queue host that sets none.
|
||||
name = "a definition of a removed callout option fails evaluation, naming the option and the secret store";
|
||||
ok =
|
||||
let
|
||||
refused =
|
||||
m: path:
|
||||
lib.any (
|
||||
a:
|
||||
!a.assertion
|
||||
&& lib.hasInfix "services.hyperhive.${path}" a.message
|
||||
&& lib.hasInfix "live in the swarm secret store" a.message
|
||||
) m.assertions;
|
||||
setting = {
|
||||
"deploy.nats.autoGenerateCallout" = hive { deploy.nats.autoGenerateCallout = true; };
|
||||
"swarm.nats.autoGenerateCallout" = hive { swarm.nats.autoGenerateCallout = true; };
|
||||
"deploy.nats.calloutUserPublicKey" = hive { deploy.nats.calloutUserPublicKey = "U"; };
|
||||
"deploy.nats.calloutIssuerPublicKey" = hive { deploy.nats.calloutIssuerPublicKey = "A"; };
|
||||
"deploy.nats.calloutUserSeedFile" = hive { deploy.nats.calloutUserSeedFile = "/run/u.seed"; };
|
||||
"swarm.nats.calloutIssuerSeedFile" = hive { swarm.nats.calloutIssuerSeedFile = "/run/i.seed"; };
|
||||
};
|
||||
in
|
||||
lib.all (path: refused setting.${path} path) (lib.attrNames setting)
|
||||
&& !(lib.any (path: refused natsWithStoreIdentity path) (lib.attrNames setting));
|
||||
}
|
||||
{
|
||||
# The responder's OIDC secret has one route, the store, under the
|
||||
|
|
|
|||
|
|
@ -43,7 +43,6 @@ let
|
|||
# The queue on a host whose store is elsewhere: no local policy unit.
|
||||
queueNoStore = hive {
|
||||
deploy.nats.enable = true;
|
||||
deploy.nats.autoGenerateCallout = true;
|
||||
};
|
||||
|
||||
# The store's host with the hive's own reader pointed somewhere else.
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# `checks.script-test-nats-render-config` — runs the queue server's
|
||||
# `ExecStartPre`, ../host-modules/swarm-nats.nix's `renderConfig`, which writes
|
||||
# the callout xkey into the config `nats-server` reads. The cases are in
|
||||
# ./nats-render-config.sh.
|
||||
# the callout's three public keys into the config `nats-server` reads. The
|
||||
# cases are in ./nats-render-config.sh.
|
||||
#
|
||||
# What runs is the rendered script. The one edit is its runtime directory,
|
||||
# moved under the build directory because the sandbox has no writable
|
||||
|
|
@ -27,11 +27,6 @@ let
|
|||
|
||||
queue = hive {
|
||||
deploy.nats.enable = true;
|
||||
deploy.nats.autoGenerateCallout = false;
|
||||
deploy.nats.calloutUserPublicKey = "UTESTUSERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
|
||||
deploy.nats.calloutIssuerPublicKey = "ATESTISSUERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
|
||||
deploy.nats.calloutUserSeedFile = "/run/secrets/nats-user.seed";
|
||||
deploy.nats.calloutIssuerSeedFile = "/run/secrets/nats-issuer.seed";
|
||||
deploy.nats.authPackage = pkgs.emptyDirectory;
|
||||
deploy.nats.authBaoClientCertFile = "/etc/pki/nats-auth.pem";
|
||||
deploy.nats.authBaoClientKeyFile = "/etc/pki/nats-auth-key.pem";
|
||||
|
|
|
|||
|
|
@ -11,8 +11,11 @@ fail() {
|
|||
failures=$((failures + 1))
|
||||
}
|
||||
|
||||
# The shape the script checks for, X and 55 base32 characters.
|
||||
# The shapes the script checks for: the key type's letter and 55 base32
|
||||
# characters.
|
||||
key=XAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
|
||||
issuer=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
|
||||
user=UAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
|
||||
|
||||
setup() {
|
||||
case=$1
|
||||
|
|
@ -27,31 +30,60 @@ setup() {
|
|||
if ! grep -qxF "d=$rt" "$dir/render"; then fail "the rewrite did not reach the script's runtime directory"; fi
|
||||
}
|
||||
|
||||
# All three public keys, as the fetch unit leaves them.
|
||||
fetched() {
|
||||
printf '%s\n' "$key" >"$rt/callout-xkey.pub"
|
||||
printf '%s\n' "$issuer" >"$rt/issuer.pub"
|
||||
printf '%s\n' "$user" >"$rt/callout-user.pub"
|
||||
}
|
||||
|
||||
go() {
|
||||
count=$((count + 1))
|
||||
"$dir/render" >"$dir/out" 2>"$dir/err"
|
||||
rc=$?
|
||||
}
|
||||
|
||||
setup "a fetched key is rendered into auth_callout.xkey, and the server's config includes it"
|
||||
printf '%s' "$key" >"$rt/callout-xkey.pub"
|
||||
refused() {
|
||||
go
|
||||
if [ "$rc" -eq 0 ]; then fail "exited 0"; fi
|
||||
if [ -e "$rt/callout.conf" ] || [ -e "$rt/nats.conf" ]; then fail "a config was written"; fi
|
||||
}
|
||||
|
||||
setup "the fetched keys are rendered into auth_callout and the callout account, and the server's config includes them"
|
||||
fetched
|
||||
go
|
||||
if [ "$rc" -ne 0 ]; then fail "exit $rc: $(cat "$dir/err")"; fi
|
||||
if ! jq -e --arg k "$key" '.authorization.auth_callout.xkey == $k' "$rt/callout.conf" >/dev/null; then
|
||||
fail "callout.conf does not carry the key as auth_callout.xkey"
|
||||
fail "callout.conf does not carry the xkey as auth_callout.xkey"
|
||||
fi
|
||||
if ! jq -e --arg k "$issuer" '.authorization.auth_callout.issuer == $k' "$rt/callout.conf" >/dev/null; then
|
||||
fail "callout.conf does not carry the issuer key as auth_callout.issuer"
|
||||
fi
|
||||
if ! jq -e --arg k "$user" '.authorization.auth_callout.auth_users == [$k] and .accounts.AUTH.users == [{nkey: $k}]' "$rt/callout.conf" >/dev/null; then
|
||||
fail "callout.conf does not carry the user key as the one exempt user and the callout account's one member"
|
||||
fi
|
||||
if ! grep -qxF 'include "callout.conf"' "$rt/nats.conf"; then fail "nats.conf does not include callout.conf"; fi
|
||||
|
||||
setup "with no key file the start fails and writes no config"
|
||||
go
|
||||
if [ "$rc" -eq 0 ]; then fail "exited 0"; fi
|
||||
if [ -e "$rt/callout.conf" ] || [ -e "$rt/nats.conf" ]; then fail "a config was written"; fi
|
||||
refused
|
||||
|
||||
setup "a file that is not a public xkey fails the start and writes no config"
|
||||
printf '%s' "not-a-key" >"$rt/callout-xkey.pub"
|
||||
go
|
||||
if [ "$rc" -eq 0 ]; then fail "exited 0"; fi
|
||||
if [ -e "$rt/callout.conf" ] || [ -e "$rt/nats.conf" ]; then fail "a config was written"; fi
|
||||
for file in callout-xkey.pub issuer.pub callout-user.pub; do
|
||||
setup "with no $file the start fails and writes no config"
|
||||
fetched
|
||||
rm "$rt/$file"
|
||||
refused
|
||||
|
||||
setup "a $file that is not its kind of public key fails the start and writes no config"
|
||||
fetched
|
||||
printf '%s' "not-a-key" >"$rt/$file"
|
||||
refused
|
||||
done
|
||||
|
||||
# Each key in the other's file: right shape, wrong letter.
|
||||
setup "an issuer key where the user key belongs fails the start and writes no config"
|
||||
fetched
|
||||
printf '%s' "$issuer" >"$rt/callout-user.pub"
|
||||
refused
|
||||
|
||||
if [ "$failures" -ne 0 ]; then
|
||||
echo "script-test-nats-render-config: $failures of $count cases broke" >&2
|
||||
|
|
|
|||
Loading…
Reference in a new issue