967 lines
45 KiB
Nix
967 lines
45 KiB
Nix
{
|
||
lib,
|
||
config,
|
||
pkgs,
|
||
...
|
||
}:
|
||
let
|
||
cfg = config.services.hyperhive.deploy.hive-controller.tls;
|
||
hyperhiveCfg = config.services.hyperhive;
|
||
gatewayCfg = config.services.hyperhive.gateway;
|
||
domain = hyperhiveCfg.domain;
|
||
baoCfg = hyperhiveCfg.swarm.bao;
|
||
baoDeploy = hyperhiveCfg.deploy.bao;
|
||
|
||
# The endpoint the services leaf comes out of. Read off the store's own
|
||
# options rather than spelled here: ./swarm-bao.nix enables the mount,
|
||
# generates its root and writes the grant that names this exact path, and a
|
||
# second module composing its own is how the grant and the caller stop
|
||
# agreeing.
|
||
baoServicesPkiMount = baoDeploy.servicesPkiMountPath;
|
||
baoServicesPkiRole = baoDeploy.servicesPkiRoleName;
|
||
|
||
# The swarm service names THIS host fronts a vhost for. Every one of them
|
||
# is inside the swarm-wide set that ./swarm-bao.nix narrows
|
||
# `pki/roles/swarm-services` to, so the leaf stays issuable while
|
||
# carrying no name this gateway does not serve.
|
||
localServiceDomains = cfg.localServiceDomains;
|
||
|
||
# The one name the request's `common_name` carries; the rest ride as
|
||
# SANs. Not `builtins.head`: a host fronting no swarm service renders
|
||
# this script too — its unit exits on `want_svc` long before running
|
||
# the request, but a bare `head []` would have failed the EVALUATION.
|
||
leafCommonName = lib.optionalString (localServiceDomains != [ ]) (
|
||
builtins.head localServiceDomains
|
||
);
|
||
|
||
# Whether the gateway's start waits for the services leaf. Not when the
|
||
# store's own browser UI is the only swarm name this host fronts: that
|
||
# host's nginx also carries the stream passthrough every reader dials to
|
||
# reach the store, and waiting on a login to that store would take that
|
||
# path down whenever it is sealed. There the gateway starts on the hive
|
||
# leaf, or on a host without a hive on a throwaway self-signed pair
|
||
# (./hive-gateway/default.nix), and the issuing script re-imports the
|
||
# services leaf into the running nginx once it lands.
|
||
#
|
||
# Nor on a host fronting no swarm name: it asks the store for the root
|
||
# alone, which nginx never reads, and a sealed store would otherwise hold
|
||
# its gateway down.
|
||
gatewayWaitsForServicesLeaf =
|
||
localServiceDomains != [ ] && localServiceDomains != [ baoCfg.ui.domain ];
|
||
|
||
# The hive leaf, `hive-tls-resign` and the hive identity it needs exist only
|
||
# where a hive runs. A gateway host without one serves the swarm-services
|
||
# leaf in their place (./hive-gateway/default.nix).
|
||
runsHive = hyperhiveCfg.deploy.hive-controller.enable;
|
||
|
||
# The gateway's own leaf is only issued when the gateway actually serves a
|
||
# self-signed cert: the gateway must run here and be in self-signed mode.
|
||
# The self-signed condition is the gateway module's single source of truth
|
||
# (`gateway.useSelfSigned`): true when neither an operator cert
|
||
# (`tls.certDir`) nor ACME is set.
|
||
active = gatewayCfg.enable && gatewayCfg.useSelfSigned;
|
||
|
||
# The role this hive's gateway leaf is issued through. ./swarm-bao.nix
|
||
# writes one per hive in the swarm directory, under the same prefix, and
|
||
# narrows it to that hive's domain, `*.` that domain and its dashboard name.
|
||
# So the leaf carries those names and no others: a configured service name
|
||
# goes on the swarm-services leaf below, and asking this role for one is a
|
||
# refusal.
|
||
hiveLeafRole =
|
||
if hyperhiveCfg.hiveName == null then
|
||
null
|
||
else
|
||
"${baoDeploy.hiveLeafPkiRolePrefix}-${hyperhiveCfg.hiveName}";
|
||
|
||
# Everything after the first rides as a SAN.
|
||
hiveLeafNames = [
|
||
domain
|
||
"*.${domain}"
|
||
]
|
||
++ lib.optional (hyperhiveCfg.swarm.dashboardDomain != null) hyperhiveCfg.swarm.dashboardDomain;
|
||
|
||
showPath = p: if p == null then "unset" else p;
|
||
|
||
# Fails the unit, naming both options, when an identity is unset or a file
|
||
# is unreadable.
|
||
requireIdentity =
|
||
{
|
||
holder,
|
||
what,
|
||
certFile,
|
||
keyFile,
|
||
certOption,
|
||
keyOption,
|
||
unsetHint ? "",
|
||
}:
|
||
purpose:
|
||
let
|
||
options = "${certOption} (${showPath certFile}) and ${keyOption} (${showPath keyFile})";
|
||
in
|
||
if certFile != null && keyFile != null then
|
||
''
|
||
missing=""
|
||
for f in ${
|
||
lib.escapeShellArgs [
|
||
certFile
|
||
keyFile
|
||
]
|
||
}; do
|
||
[ -r "$f" ] || missing="$missing $f"
|
||
done
|
||
if [ -n "$missing" ]; then
|
||
echo "${holder} has no usable ${what}, so it cannot ${purpose}: cannot read$missing." >&2
|
||
echo "place the files named by ${options}." >&2
|
||
exit 1
|
||
fi
|
||
''
|
||
else
|
||
''
|
||
echo "${holder} has no ${what}, so it cannot ${purpose}." >&2
|
||
echo "set ${options} and place both files.${unsetHint}" >&2
|
||
exit 1
|
||
'';
|
||
|
||
# The hive's own store identity: its subject is
|
||
# `<hiveLeafCommonNamePrefix>-<hiveName>`, and its policy may issue from
|
||
# `hiveLeafRole` alone. It also carries the root fetch's handshake. Every
|
||
# hive must hold one.
|
||
haveHiveIdentity = cfg.baoHiveIdentityCertFile != null && cfg.baoHiveIdentityKeyFile != null;
|
||
requireHiveIdentity = requireIdentity {
|
||
holder = "this hive";
|
||
what = "store identity";
|
||
certFile = cfg.baoHiveIdentityCertFile;
|
||
keyFile = cfg.baoHiveIdentityKeyFile;
|
||
certOption = "services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile";
|
||
keyOption = "services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityKeyFile";
|
||
};
|
||
|
||
# The swarm-wide services-issuer identity `swarm-services-cert` logs in
|
||
# with to issue the services leaf, whether or not a hive runs here. Its
|
||
# subject is `servicesIssuerCommonName`, and its policy may issue from
|
||
# `pki/issue/swarm-services` alone.
|
||
servicesIssuerCert = baoDeploy.servicesIssuerClientCertFile;
|
||
servicesIssuerKey = baoDeploy.servicesIssuerClientKeyFile;
|
||
requireServicesIssuer = requireIdentity {
|
||
holder = "this host";
|
||
what = "services-issuer identity";
|
||
certFile = servicesIssuerCert;
|
||
keyFile = servicesIssuerKey;
|
||
certOption = "services.hyperhive.deploy.bao.servicesIssuerClientCertFile";
|
||
keyOption = "services.hyperhive.deploy.bao.servicesIssuerClientKeyFile";
|
||
unsetHint = " Its subject must be CN=${baoDeploy.servicesIssuerCommonName}.";
|
||
};
|
||
|
||
# Shared by every unit that decides whether to re-issue a leaf. It
|
||
# lives here rather than in one of them because the guards have to
|
||
# agree: they answer the same question for different leaves
|
||
# (`hive-tls-resign` for the hive leaf, at boot, on the deploy and from
|
||
# its timer; `swarm-services-cert` for the services leaf). A rule
|
||
# implemented in one and not the others is worse than one implemented
|
||
# in none — it looks fixed and only fires on whichever path you did not
|
||
# take, which is exactly how a corrected `serviceDomains` still served a
|
||
# stale leaf.
|
||
#
|
||
# Expects `$d` (state dir) to be set; defines `$leaf`-adjacent names and
|
||
# `covers`.
|
||
leafCoverage = ''
|
||
svcleaf="$d/swarm-services.pem"
|
||
svcroot="$d/swarm-services-root.pem"
|
||
hiveNames=${lib.escapeShellArg (lib.concatStringsSep " " hiveLeafNames)}
|
||
svcNames=${lib.escapeShellArg (lib.concatStringsSep " " localServiceDomains)}
|
||
|
||
# A host fronting none of the swarm's service names has no services
|
||
# leaf to hold, and its absence there is the correct state rather
|
||
# than a stale one — its own vhosts are covered by the hive leaf
|
||
# above. Anywhere else it is expected: the leaf comes from the
|
||
# secret store's `pki` mount now, not from a sub-CA that exists on
|
||
# one host in the swarm, so "this host cannot issue it" is no longer
|
||
# one of the answers.
|
||
want_svc=${if localServiceDomains == [ ] then "0" else "1"}
|
||
|
||
# Expiry is not the only way a leaf goes wrong. One signed when the
|
||
# configured name set was smaller stays valid for its whole lifetime
|
||
# while omitting every name added since — so a config change can
|
||
# evaluate, build and deploy cleanly while the gateway keeps serving a
|
||
# certificate that does not cover the new service.
|
||
#
|
||
# Reads the names out of the CERTIFICATE, not a sidecar file: the pem
|
||
# is what nginx serves, and a bookkeeping file drifts from it the
|
||
# moment anyone replaces a leaf by hand.
|
||
covers() { # $1 = leaf, $2 = space-separated names it must carry
|
||
[ -s "$1" ] || return 1
|
||
_have="$(openssl x509 -in "$1" -noout -ext subjectAltName 2>/dev/null \
|
||
| tr ',' '\n' | sed -n 's/^[[:space:]]*DNS:\(.*\)$/\1/p' || true)"
|
||
[ -n "$_have" ] || return 1
|
||
_missing=0
|
||
# ⚠️ The hive leaf carries `*.<domain>`; without this the shell
|
||
# expands it against the cwd and the comparison silently tests a
|
||
# filename instead of a name.
|
||
set -f
|
||
for _n in $2; do
|
||
printf '%s\n' "$_have" | grep -qxF "$_n" || _missing=1
|
||
done
|
||
set +f
|
||
[ "$_missing" = 0 ]
|
||
}
|
||
|
||
# True when the hive leaf is present and carries its configured
|
||
# names. Expiry is the callers' own business — they use different
|
||
# windows.
|
||
#
|
||
# The services leaf is not part of this: `swarm-services-cert` holds
|
||
# the same rule for it, using the `covers` helper above so the two
|
||
# cannot drift.
|
||
leavesCoverNames() {
|
||
covers "$leaf" "$hiveNames"
|
||
}
|
||
'';
|
||
|
||
# What `swarm-services-cert` and `swarm-services-cert-renew` both run the
|
||
# issuance with. Bound here rather than read back off the first unit: its
|
||
# merged `path` already carries systemd's defaults, so the second would
|
||
# render a different `PATH` and fail to merge.
|
||
servicesCertPath = [
|
||
baoDeploy.package
|
||
pkgs.jq
|
||
pkgs.openssl
|
||
pkgs.coreutils
|
||
pkgs.systemd
|
||
# `cmp`, which is NOT in coreutils. Without it the root-changed test
|
||
# below is `command not found` — 127, invisible under `if !`, and
|
||
# therefore always "changed", so the bundle rebuild it guards fired
|
||
# on every issuance instead of on a new root.
|
||
pkgs.diffutils
|
||
# `flock`, for the lock at the top of the script.
|
||
pkgs.util-linux
|
||
# The root-only fetch on a host fronting no swarm name.
|
||
pkgs.curl
|
||
];
|
||
# One attempt of the issuance: up to 10 minutes on the lock at the top of
|
||
# the script, then the store calls. A oneshot has no start timeout of its
|
||
# own, and an attempt that never ends never fails into a retry.
|
||
servicesCertTimeoutStartSec = "20min";
|
||
servicesCertEnvironment = {
|
||
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||
}
|
||
// lib.optionalAttrs (servicesIssuerCert != null) {
|
||
BAO_CLIENT_CERT = servicesIssuerCert;
|
||
}
|
||
// lib.optionalAttrs (servicesIssuerKey != null) {
|
||
BAO_CLIENT_KEY = servicesIssuerKey;
|
||
}
|
||
# Absent means the system trust store, which is what a deployment
|
||
# with a real CA wants and what a self-signed one must not be left
|
||
# with.
|
||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||
BAO_CACERT = baoDeploy.serverCaFile;
|
||
};
|
||
|
||
# The root-only fetch. Same store and server CA as `servicesCertEnvironment`,
|
||
# but the hive's own identity: a host fronting no swarm name holds no
|
||
# services-issuer pair.
|
||
storeRootCurlTls = lib.escapeShellArgs (
|
||
lib.optionals (baoDeploy.serverCaFile != null) [
|
||
"--cacert"
|
||
baoDeploy.serverCaFile
|
||
]
|
||
++ [
|
||
"--cert"
|
||
cfg.baoHiveIdentityCertFile
|
||
"--key"
|
||
cfg.baoHiveIdentityKeyFile
|
||
]
|
||
);
|
||
|
||
# What `hive-tls-resign` issues the hive leaf with: the services leaf's
|
||
# store and server CA, presenting the hive's own identity.
|
||
hiveLeafEnvironment = {
|
||
inherit (servicesCertEnvironment) BAO_ADDR;
|
||
}
|
||
// lib.optionalAttrs (cfg.baoHiveIdentityCertFile != null) {
|
||
BAO_CLIENT_CERT = cfg.baoHiveIdentityCertFile;
|
||
}
|
||
// lib.optionalAttrs (cfg.baoHiveIdentityKeyFile != null) {
|
||
BAO_CLIENT_KEY = cfg.baoHiveIdentityKeyFile;
|
||
}
|
||
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
||
BAO_CACERT = baoDeploy.serverCaFile;
|
||
};
|
||
storeRetry = import ./lib/store-retry.nix { };
|
||
in
|
||
{
|
||
# Host-side TLS for the self-signed gateway mode.
|
||
#
|
||
# The gateway serves a short-lived **leaf** issued by the secret store's
|
||
# `pki` mount through this hive's own role, and consumers (agents,
|
||
# federation peers) anchor on that mount's root, which ships in
|
||
# `trust-bundle.pem`. Leaf rotation never re-breaks them, and nothing here
|
||
# generates or signs with a CA key. See
|
||
# `docs/networking/gateway.md` ("Self-signed TLS") and docs/swarm/ca.md.
|
||
|
||
options.services.hyperhive.deploy.hive-controller.tls = {
|
||
stateDir = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "/var/lib/hive-tls";
|
||
description = ''
|
||
Host directory holding the gateway leaf and trust bundle for the
|
||
self-signed gateway mode: `gateway.pem` / `gateway-key.pem` (the
|
||
leaf the store issues and nginx serves) and `trust-bundle.pem` (the
|
||
anchor agents and federation peers trust, the store's `pki` root).
|
||
'';
|
||
};
|
||
|
||
baoHiveIdentityCertFile = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/var/lib/swarm-bao-pki/hive-leaf.pem";
|
||
description = ''
|
||
This hive's own store identity. `hive-tls-resign` presents it to ask
|
||
the secret store for the gateway leaf, and `swarm-services-cert` on a
|
||
hive fronting no swarm name presents it for the handshake of the root
|
||
fetch.
|
||
|
||
Its subject has to be
|
||
`<services.hyperhive.deploy.bao.hiveLeafCommonNamePrefix>-<hiveName>`;
|
||
that cert-auth role's policy may issue from this hive's gateway-leaf
|
||
role and nothing else.
|
||
|
||
Every hive with a self-signed gateway needs it: without it both units
|
||
fail, naming this option and
|
||
{option}`services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityKeyFile`.
|
||
On the host that runs the store a glue module supplies the leaf as a
|
||
`mkDefault`; on every other host an operator places it.
|
||
|
||
A path, never a value.
|
||
'';
|
||
};
|
||
|
||
baoHiveIdentityKeyFile = lib.mkOption {
|
||
type = lib.types.nullOr lib.types.str;
|
||
default = null;
|
||
example = "/var/lib/swarm-bao-pki/hive-leaf-key.pem";
|
||
description = ''
|
||
Private key for
|
||
{option}`services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile`.
|
||
Both or neither — a certificate with no key authenticates nothing.
|
||
'';
|
||
};
|
||
|
||
localServiceDomains = lib.mkOption {
|
||
type = lib.types.listOf lib.types.str;
|
||
readOnly = true;
|
||
internal = true;
|
||
description = ''
|
||
Read-only: the names in
|
||
{option}`services.hyperhive.swarm.serviceDomains` **this host
|
||
actually fronts**, which is what its services leaf may carry. The
|
||
swarm-wide set stays what the sub-CA is name-constrained to and what
|
||
the store's issuing role permits; narrowing the leaf is what stops a
|
||
private key on one gateway presenting a valid certificate for
|
||
services that host does not serve and never has.
|
||
|
||
Read off the rendered `services.nginx.virtualHosts` rather than off
|
||
the per-service deploy flags: those flags are what the vhosts are
|
||
already guarded on, so a second reading of them is a copy that
|
||
drifts from the thing it claims to describe. Filtering
|
||
`swarm.serviceDomains` rather than assembling a list beside it is
|
||
that option's own constraint, and it applies here for the same
|
||
reason.
|
||
'';
|
||
};
|
||
};
|
||
|
||
config = lib.mkMerge [
|
||
{
|
||
# Outside `active`: the option has no default, so reading it on a host
|
||
# whose gateway is not self-signed would otherwise throw.
|
||
services.hyperhive.deploy.hive-controller.tls.localServiceDomains = lib.filter (
|
||
d: config.services.nginx.virtualHosts ? ${d}
|
||
) hyperhiveCfg.swarm.serviceDomains;
|
||
}
|
||
|
||
(lib.mkIf active {
|
||
# The state dir and the trust bundle. Never waits on the store: the
|
||
# store's container is ordered after this unit, which writes the
|
||
# bundle it bind-mounts, so the leaf is `hive-tls-resign`'s.
|
||
systemd.services.hive-tls-ca = {
|
||
description = "Assemble the hive TLS trust bundle (self-signed mode)";
|
||
wantedBy = [ "multi-user.target" ];
|
||
# The consumer is `hive-gateway-self-signed-cert`, which copies the
|
||
# leaf out of the state dir this unit creates, and which nginx in
|
||
# turn `Requires=`. Order against the unit that reads the dir.
|
||
before = [ "hive-gateway-self-signed-cert.service" ];
|
||
requiredBy = [ "hive-gateway-self-signed-cert.service" ];
|
||
# `swarm-services-cert` is NOT ordered against this unit in either
|
||
# direction, and the absence is load-bearing — see the cycle spelled
|
||
# out at that unit. It runs after the store's container, the container
|
||
# runs after this unit, so an edge back to here is a loop and systemd
|
||
# resolves a loop by dropping a job.
|
||
#
|
||
# What that costs: the trust bundle written at the end of this script
|
||
# is the services root, so on a first boot it is written before that
|
||
# file exists and is empty until the script runs again. The `if [ -s
|
||
# "$svcroot" ]` guard is what makes that state legal rather than fatal,
|
||
# and `swarm-services-cert` reloads this unit once the root it wrote
|
||
# is new — so the bundle converges instead of being a boot behind.
|
||
#
|
||
# ⚠️ A switch that changes this unit reloads it, running the same
|
||
# script, rather than stopping and starting it. A stop propagates to
|
||
# every unit that `Requires=` this one: the store's container, which
|
||
# comes back sealed under the `shamir` seal, every container that
|
||
# binds the bundle, and every `<name>-ca-bundle` unit with its
|
||
# consumers. A reload rewrites the bundle in place and touches none of
|
||
# them; each `<name>-ca-bundle` unit watches the file it is built from
|
||
# and rebuilds itself and its consumers (`lib/hive-ca-trust.nix`).
|
||
reloadIfChanged = true;
|
||
reload = config.systemd.services.hive-tls-ca.script;
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
UMask = "0077";
|
||
# Pin the journal identity (else it's the `script` store-path wrapper).
|
||
SyslogIdentifier = "hive-tls-ca";
|
||
};
|
||
script = ''
|
||
set -euo pipefail
|
||
d=${lib.escapeShellArg cfg.stateDir}
|
||
install -d -m 0755 "$d"
|
||
|
||
svcroot="$d/swarm-services-root.pem"
|
||
|
||
# --- Trust bundle: what a consumer must TRUST, the store's `pki`
|
||
# root, which both the gateway leaf and the swarm-services leaf
|
||
# chain to (docs/swarm/ca.md).
|
||
#
|
||
# Written IN PLACE, never renamed into position — containers
|
||
# bind-mount this file and a bind mount follows the inode, so a
|
||
# rename leaves every consumer holding the old one.
|
||
bundle="$d/trust-bundle.pem"
|
||
if [ -s "$svcroot" ]; then
|
||
cat "$svcroot" > "$bundle"
|
||
else
|
||
echo "the store's root $svcroot is missing or empty, so $bundle is empty and every *-ca-bundle unit fails until swarm-services-cert fetches it" >&2
|
||
: > "$bundle"
|
||
fi
|
||
chmod 0644 "$bundle"
|
||
'';
|
||
};
|
||
|
||
# The gateway leaf: issued by the secret store's `pki` mount through
|
||
# this hive's own role, at boot, on every deploy and from the daily
|
||
# timer below, whenever it is missing, past half its lifetime, missing
|
||
# a configured name, not chained to the store's root, or not matching
|
||
# its key.
|
||
#
|
||
# ⚠️ Nothing orders the gateway against this unit, which is what lets a
|
||
# hive holding a valid leaf start its gateway with the store sealed or
|
||
# unreachable. Only a hive with no leaf yet has a gateway that waits
|
||
# for the store: its cert import fails until this unit lands a leaf
|
||
# and starts nginx.
|
||
#
|
||
# Failure leaves the current leaf alone: every file the script writes
|
||
# is a `.new` until the store has answered with both fields.
|
||
#
|
||
# Propagation mechanism: nginx serves a *copy* of the leaf, written by
|
||
# `hive-gateway-self-signed-cert` at the mode nginx can read. Issuing
|
||
# the source therefore changes nothing on its own — the copy has to be
|
||
# remade and nginx reloaded, which is what the calls at the end do.
|
||
# Neither of the running-nginx calls is `|| true`: a swallowed
|
||
# propagation failure means the leaf rotates on disk while nginx keeps
|
||
# serving the old copy until it expires, with this unit reporting
|
||
# success the whole time.
|
||
systemd.services.hive-tls-resign = lib.mkIf runsHive {
|
||
description = "Issue or renew the gateway TLS leaf from the secret store's PKI";
|
||
wantedBy = [ "multi-user.target" ];
|
||
# The store's container and its granting unit where they run here;
|
||
# ignored elsewhere, as for `swarm-services-cert`, which writes the
|
||
# store's root this unit checks the leaf against.
|
||
after = [
|
||
"hive-tls-ca.service"
|
||
"container@${baoCfg.machine}.service"
|
||
"swarm-bao-pki.service"
|
||
"swarm-bao-hive-leaf-policy.service"
|
||
"swarm-services-cert.service"
|
||
];
|
||
wants = [ "container@${baoCfg.machine}.service" ];
|
||
path = [
|
||
baoDeploy.package
|
||
pkgs.jq
|
||
pkgs.openssl
|
||
pkgs.coreutils
|
||
pkgs.systemd
|
||
];
|
||
environment = hiveLeafEnvironment;
|
||
# ./lib/store-retry.nix: the login below fails for as long as the
|
||
# store is sealed or down.
|
||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||
serviceConfig = storeRetry.serviceConfig // {
|
||
Type = "oneshot";
|
||
UMask = "0077";
|
||
SyslogIdentifier = "hive-tls-resign";
|
||
# One attempt. A oneshot has no start timeout of its own, and an
|
||
# attempt that never ends never fails into a retry.
|
||
TimeoutStartSec = "10min";
|
||
};
|
||
script = ''
|
||
set -euo pipefail
|
||
d=${lib.escapeShellArg cfg.stateDir}
|
||
install -d -m 0755 "$d"
|
||
leaf="$d/gateway.pem"
|
||
leafk="$d/gateway-key.pem"
|
||
|
||
fail() {
|
||
echo "$1" >&2
|
||
if [ -s "$leaf" ]; then
|
||
echo "the current gateway leaf at $leaf stays in place" >&2
|
||
fi
|
||
exit 1
|
||
}
|
||
|
||
# Checked before the leaf is: a hive without its identity is red
|
||
# even while its current leaf is still valid.
|
||
${
|
||
if hiveLeafRole == null then
|
||
''fail "services.hyperhive.hiveName is unset, so this hive has no gateway-leaf role to issue from."''
|
||
else
|
||
requireHiveIdentity "ask the store for its gateway leaf"
|
||
}
|
||
|
||
${leafCoverage}
|
||
|
||
# Half the lifetime the role grants, as for the services leaf, so
|
||
# the daily timer has half the lifetime to retry through a sealed
|
||
# store.
|
||
renewat=$(( ${toString baoDeploy.servicesPkiLeafTtlHours} * 3600 / 2 ))
|
||
reissue=0
|
||
{ [ -s "$leaf" ] && [ -s "$leafk" ]; } || reissue=1
|
||
openssl x509 -in "$leaf" -noout -checkend "$renewat" >/dev/null 2>&1 || reissue=1
|
||
leavesCoverNames || reissue=1
|
||
# A leaf that does not chain to the store's current root, whoever
|
||
# signed it, fails this and is re-issued.
|
||
{ [ -s "$svcroot" ] && openssl verify -CAfile "$svcroot" "$leaf" >/dev/null 2>&1; } || reissue=1
|
||
# The leaf and the key are two renames, so an interrupted rotation can
|
||
# leave a certificate beside a key it does not belong to.
|
||
leafpub="$(openssl x509 -in "$leaf" -noout -pubkey 2>/dev/null || true)"
|
||
keypub="$(openssl pkey -in "$leafk" -pubout 2>/dev/null || true)"
|
||
{ [ -n "$leafpub" ] && [ "$leafpub" = "$keypub" ]; } || reissue=1
|
||
|
||
if [ "$reissue" = 0 ]; then
|
||
echo "gateway leaf valid, matching its key, chained to the store's root and covering the configured names — leaving it alone"
|
||
exit 0
|
||
fi
|
||
|
||
err="$(mktemp)"
|
||
resp="$(mktemp "$d/gateway.json.XXXXXX")"
|
||
trap 'rm -f "$err" "$resp" "$leaf.new" "$leafk.new"' EXIT
|
||
|
||
# Cert auth is a login, not a transport setting; `-token-only`
|
||
# skips the token helper, for the reason `swarm-services-cert` gives.
|
||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||
fail "could not log in to the swarm secret store with this hive's identity: $(cat "$err")"
|
||
fi
|
||
export BAO_TOKEN
|
||
|
||
echo "requesting the gateway leaf from the store"
|
||
if ! bao write -format=json \
|
||
${lib.escapeShellArg "${baoServicesPkiMount}/issue/${toString hiveLeafRole}"} \
|
||
common_name=${lib.escapeShellArg domain} \
|
||
alt_names=${lib.escapeShellArg (lib.concatStringsSep "," (lib.tail hiveLeafNames))} \
|
||
> "$resp" 2>"$err"; then
|
||
fail "the store refused to issue the gateway leaf: $(cat "$err")"
|
||
fi
|
||
|
||
# One issuance, split, for the reasons `swarm-services-cert` gives:
|
||
# the key never enters an argv, and `// empty` keeps a missing field
|
||
# from becoming the four characters `null`. The leaf alone, without
|
||
# its issuer: the issuer is the self-signed root in the bundle.
|
||
umask 077
|
||
jq -r '.data.private_key // empty' < "$resp" > "$leafk.new"
|
||
jq -r '.data.certificate // empty' < "$resp" > "$leaf.new"
|
||
for f in "$leafk.new" "$leaf.new"; do
|
||
[ -s "$f" ] || fail "the store's response was missing a field: $f is empty"
|
||
done
|
||
|
||
mv -f "$leaf.new" "$leaf"
|
||
mv -f "$leafk.new" "$leafk"
|
||
chmod 0644 "$leaf"
|
||
chmod 0600 "$leafk"
|
||
|
||
# A gateway already running gets the copy remade and a reload. One
|
||
# that is not running — a hive that had no leaf — gets started,
|
||
# `--no-block` so a start job still queued absorbs the request.
|
||
if systemctl is-active --quiet nginx.service; then
|
||
echo "gateway leaf rotated — re-importing and reloading nginx"
|
||
systemctl restart hive-gateway-self-signed-cert.service
|
||
systemctl reload nginx.service
|
||
elif [ "$(systemctl show -P LoadState nginx.service)" = loaded ]; then
|
||
echo "gateway leaf landed with nginx not running — starting it"
|
||
systemctl reset-failed nginx.service
|
||
systemctl start --no-block nginx.service
|
||
fi
|
||
'';
|
||
};
|
||
|
||
# The swarm-services leaf, issued by the secret store's `pki` mount.
|
||
#
|
||
# The leaf chains straight to the mount's root, with no intermediate:
|
||
# the store's `pki/roles/swarm-services` narrows the names with an
|
||
# `allowed_domains` list enforced at issue time, so there is no x509
|
||
# `nameConstraints` for an intermediate to carry.
|
||
#
|
||
# The names it may carry are not this unit's to assert: the role
|
||
# refuses anything outside `allowed_domains`, read from the
|
||
# swarm-wide `swarm.serviceDomains` that the SANs below are a
|
||
# per-host SUBSET of. A mismatch is a refusal from the store naming
|
||
# the offending name, not a certificate quietly issued for something
|
||
# nobody configured.
|
||
systemd.services.swarm-services-cert = {
|
||
description = "Issue the swarm-services TLS leaf from the secret store's PKI";
|
||
wantedBy = [ "multi-user.target" ];
|
||
# ⚠️ Where the gateway waits for this leaf, the store being up is a
|
||
# precondition of the gateway's certificate, and `requiredBy` is
|
||
# what makes a store that never comes up an outage that says so
|
||
# instead of a gateway serving the wrong name.
|
||
#
|
||
# 🩸 NOT `Before=hive-tls-ca.service`, however much the trust bundle
|
||
# wants it. That edge closed a cycle — the store's container is
|
||
# `After=hive-tls-ca` (it serves the material that unit writes), and
|
||
# this unit is `After=` the container — so systemd broke the loop the
|
||
# only way it can, by deleting a job:
|
||
#
|
||
# Found ordering cycle: container@swarm-bao.service/start after
|
||
# hive-tls-ca.service/start after swarm-services-cert.service/start
|
||
# Job swarm-services-cert.service/start deleted to break ordering cycle
|
||
#
|
||
# With the job gone the leaf was never issued, `Requires=` took the
|
||
# gateway's cert import down with it, and nginx never started at all —
|
||
# so the swarm served nothing rather than serving the wrong name. The
|
||
# bundle reaches the root the other way round, at the end of the script
|
||
# below: it reloads `hive-tls-ca` when the root CHANGED, which is the
|
||
# same path that already covered a store coming up hours late.
|
||
before = lib.optional gatewayWaitsForServicesLeaf "hive-gateway-self-signed-cert.service";
|
||
requiredBy = lib.optional gatewayWaitsForServicesLeaf "hive-gateway-self-signed-cert.service";
|
||
# The store's container, where it runs here. On a hive that reads a
|
||
# store hosted elsewhere no such unit exists and systemd ignores
|
||
# the name, which is the correct behaviour rather than a gap: what
|
||
# this unit actually needs is the store reachable, and on a remote
|
||
# store nothing local can order against that.
|
||
#
|
||
# `swarm-bao-pki` mints the services-issuer leaf the login below
|
||
# presents, and is absent on a host holding an operator-placed one —
|
||
# ignored there for the same reason, rather than being a dependency
|
||
# only some hosts may declare.
|
||
after = [
|
||
"container@${baoCfg.machine}.service"
|
||
"swarm-bao-pki.service"
|
||
"swarm-bao-services-issuer-policy.service"
|
||
];
|
||
wants = [ "container@${baoCfg.machine}.service" ];
|
||
path = servicesCertPath;
|
||
# ./lib/store-retry.nix: the login below fails for as long as the
|
||
# store is sealed or down.
|
||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||
serviceConfig = storeRetry.serviceConfig // {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
UMask = "0077";
|
||
SyslogIdentifier = "swarm-services-cert";
|
||
TimeoutStartSec = servicesCertTimeoutStartSec;
|
||
};
|
||
environment = servicesCertEnvironment;
|
||
script = ''
|
||
set -euo pipefail
|
||
d=${lib.escapeShellArg cfg.stateDir}
|
||
install -d -m 0755 "$d"
|
||
|
||
# `swarm-services-cert-renew` runs this same script, and two runs
|
||
# issuing at once can leave one's key beside the other's leaf. The
|
||
# second waits and then finds the leaf fresh.
|
||
exec 9>"$d/.swarm-services-cert.lock"
|
||
if ! flock -w 600 9; then
|
||
echo "another swarm-services issuance has held $d/.swarm-services-cert.lock for 10 minutes" >&2
|
||
exit 1
|
||
fi
|
||
|
||
${leafCoverage}
|
||
|
||
# No leaf to ask for, but the root still belongs in the trust
|
||
# bundle: every other hive's swarm-service names chain to it.
|
||
# `ca/pem` takes no token; the TLS client certificate is the
|
||
# store's listener's demand, not bao's, so this presents the
|
||
# hive's own identity and never logs in with it.
|
||
# A hive without a readable identity fails here, naming the
|
||
# options: every hive must hold one.
|
||
if [ "$want_svc" = 0 ]; then
|
||
${
|
||
if haveHiveIdentity then
|
||
requireHiveIdentity "fetch the store's ${baoServicesPkiMount} root"
|
||
+ ''
|
||
echo "no swarm service domains configured — fetching only the store's pki root"
|
||
err="$(mktemp)"
|
||
raw="$(mktemp "$d/swarm-services-root.XXXXXX")"
|
||
trap 'rm -f "$err" "$raw"' EXIT
|
||
if ! curl --fail --silent --show-error \
|
||
--connect-timeout 10 --max-time 30 \
|
||
${storeRootCurlTls} --output "$raw" \
|
||
"$BAO_ADDR/v1/${baoServicesPkiMount}/ca/pem" 2>"$err"; then
|
||
echo "could not read the store's ${baoServicesPkiMount} root." >&2
|
||
cat "$err" >&2
|
||
exit 1
|
||
fi
|
||
# Re-encoded rather than copied: it fails on a body that does not
|
||
# parse as a certificate, and ends the file in the newline the
|
||
# bundle's `cat` relies on.
|
||
if ! openssl x509 -in "$raw" -out "$svcroot.new" 2>"$err"; then
|
||
echo "the store's ${baoServicesPkiMount} root does not parse as a certificate." >&2
|
||
cat "$err" >&2
|
||
exit 1
|
||
fi
|
||
if cmp -s "$svcroot.new" "$svcroot"; then
|
||
rm -f "$svcroot.new"
|
||
echo "the store's pki root is unchanged"
|
||
exit 0
|
||
fi
|
||
mv -f "$svcroot.new" "$svcroot"
|
||
chmod 0644 "$svcroot"
|
||
if systemctl is-active --quiet hive-tls-ca.service; then
|
||
echo "the services root changed — rebuilding the trust bundle"
|
||
systemctl reload --no-block hive-tls-ca.service
|
||
fi
|
||
exit 0''
|
||
else
|
||
requireHiveIdentity "fetch the store's ${baoServicesPkiMount} root"
|
||
}
|
||
fi
|
||
|
||
# Re-issue when the file is missing, past half the lifetime the
|
||
# store's role grants it, or no longer carrying every configured
|
||
# name. The name check is what makes adding a swarm service take
|
||
# effect on the rebuild that added it rather than whenever the
|
||
# certificate happens to lapse.
|
||
#
|
||
# Half, as `hive-tls-resign` does for the hive leaf, so the daily
|
||
# renewal has half the lifetime to retry through a sealed store.
|
||
# Not the whole lifetime: a leaf issued with exactly that much left
|
||
# is inside it the second after, so every run re-issued.
|
||
renewat=$(( ${toString baoDeploy.servicesPkiLeafTtlHours} * 3600 / 2 ))
|
||
reissue=0
|
||
{ [ -s "$svcleaf" ] && [ -s "$d/swarm-services-key.pem" ]; } || reissue=1
|
||
openssl x509 -in "$svcleaf" -noout -checkend "$renewat" >/dev/null 2>&1 || reissue=1
|
||
covers "$svcleaf" "$svcNames" || reissue=1
|
||
[ -s "$svcroot" ] || reissue=1
|
||
# A leaf that does not chain to the root the trust bundle holds,
|
||
# whoever signed it, fails this and is re-issued.
|
||
openssl verify -CAfile "$svcroot" "$svcleaf" >/dev/null 2>&1 || reissue=1
|
||
|
||
# The ROOT's window, not the leaf's, and it is what keeps the two
|
||
# sides of this from needing to talk. The store replaces an issuer
|
||
# that has less than a leaf's window left (./swarm-bao.nix's
|
||
# regeneration guard) — so a hive testing its own copy of that same
|
||
# certificate against the same threshold, read from the same option,
|
||
# asks for a new leaf on the same activation, and gets the
|
||
# replacement root back with it. Without this a leaf stays "valid"
|
||
# while the anchor it chains to no longer exists in the mount, which
|
||
# no expiry check would ever catch.
|
||
openssl x509 -in "$svcroot" -noout -checkend ${
|
||
toString (baoDeploy.servicesPkiLeafTtlHours * 3600)
|
||
} >/dev/null 2>&1 || reissue=1
|
||
|
||
if [ "$reissue" = 0 ]; then
|
||
echo "swarm-services leaf valid and covering the configured names — leaving it alone"
|
||
exit 0
|
||
fi
|
||
|
||
${requireServicesIssuer "ask the store for the swarm-services leaf"}
|
||
|
||
# `bao`'s own message is the only thing separating an unreachable
|
||
# store from a refused identity from a role that would not issue
|
||
# these names, and this unit retries on all three — so it reports
|
||
# which one rather than asserting all three in a sentence of ours.
|
||
err="$(mktemp)"
|
||
trap 'rm -f "$err"' EXIT
|
||
|
||
# Cert auth is a login, not a transport setting: the
|
||
# `BAO_CLIENT_*` variables only decide which certificate the
|
||
# handshake presents. Without a token `bao` asks its token
|
||
# helper, and that is a `sh` this unit's `path` does not carry —
|
||
# `-token-only` answers on stdout and skips the helper.
|
||
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
||
echo "could not log in to the swarm secret store with this host's services-issuer certificate." >&2
|
||
cat "$err" >&2
|
||
exit 1
|
||
fi
|
||
export BAO_TOKEN
|
||
|
||
echo "requesting the swarm-services leaf from the store"
|
||
resp="$(mktemp "$d/swarm-services.json.XXXXXX")"
|
||
trap 'rm -f "$err" "$resp"' EXIT
|
||
|
||
# ONE call, and the response carries all three artefacts —
|
||
# certificate, private key and issuing CA. Splitting it into a
|
||
# local CSR plus `pki/sign` would buy a key that never crosses
|
||
# the wire, and cost the thing that makes this unit worth having:
|
||
# the store, not this host, decides what the certificate says.
|
||
if ! bao write -format=json \
|
||
${lib.escapeShellArg "${baoServicesPkiMount}/issue/${baoServicesPkiRole}"} \
|
||
common_name=${lib.escapeShellArg leafCommonName} \
|
||
alt_names=${lib.escapeShellArg (lib.concatStringsSep "," localServiceDomains)} \
|
||
> "$resp" 2>"$err"; then
|
||
echo "the store refused to issue the swarm-services certificate." >&2
|
||
cat "$err" >&2
|
||
exit 1
|
||
fi
|
||
|
||
# Three fields out of ONE issuance, which is why the response is
|
||
# captured and then split rather than asked for three times with
|
||
# `bao write -field=…`: each of those calls would mint a separate
|
||
# certificate, and the key would belong to a different one than
|
||
# the leaf.
|
||
#
|
||
# `jq -r` into a file the shell opened, never into a variable and
|
||
# never into an argv — the private key does not become visible in
|
||
# /proc on its way to disk.
|
||
#
|
||
# `// empty` on every one of them: `jq -r` renders a missing
|
||
# field as the four characters `null`, which is a non-empty file
|
||
# and would sail past the check below into a certificate slot.
|
||
umask 077
|
||
jq -r '.data.private_key // empty' < "$resp" > "$d/swarm-services-key.pem.new"
|
||
jq -r '.data.certificate // empty' < "$resp" > "$d/swarm-services-leaf.new"
|
||
jq -r '.data.issuing_ca // empty' < "$resp" > "$svcroot.new"
|
||
|
||
for f in "$d/swarm-services-key.pem.new" "$d/swarm-services-leaf.new" "$svcroot.new"; do
|
||
if [ ! -s "$f" ]; then
|
||
echo "the store's response was missing a field: $f is empty" >&2
|
||
exit 1
|
||
fi
|
||
done
|
||
|
||
# nginx serves this file verbatim. It carries the leaf ALONE and
|
||
# not leaf-plus-issuer, which is the one visible consequence of
|
||
# flattening the chain: the issuer here is a self-signed root, so
|
||
# appending it would ship an anchor a client either already holds
|
||
# — in which case it is redundant — or does not, in which case
|
||
# shipping it proves nothing.
|
||
mv -f "$d/swarm-services-leaf.new" "$svcleaf"
|
||
mv -f "$d/swarm-services-key.pem.new" "$d/swarm-services-key.pem"
|
||
chmod 0644 "$svcleaf"
|
||
chmod 0600 "$d/swarm-services-key.pem"
|
||
|
||
# The anchor, world-readable, beside the leaf it signed. This is
|
||
# the file an operator adds to a browser's trust store, and the
|
||
# one `hive-tls-ca` folds into `trust-bundle.pem`. Public
|
||
# certificate material only — the private half never leaves the
|
||
# store's `pki` mount, which is what `root/generate/internal`
|
||
# buys.
|
||
rootchanged=0
|
||
if ! cmp -s "$svcroot.new" "$svcroot"; then
|
||
rootchanged=1
|
||
fi
|
||
mv -f "$svcroot.new" "$svcroot"
|
||
chmod 0644 "$svcroot"
|
||
|
||
# Propagation. nginx serves a *copy* of the leaf, so re-issuing the
|
||
# source changes nothing until the copy is remade. A running
|
||
# gateway (`swarm-services-cert-renew` rotating the leaf) gets the
|
||
# copy remade and a reload. A gateway that is not running gets
|
||
# started: a start job still waiting on this unit absorbs the
|
||
# request, and a start that already ended on this unit's
|
||
# `requiredBy` has nothing else that would start it again.
|
||
#
|
||
# ⚠️ `--no-block`, and it is not a preference. This unit declares
|
||
# `Before=` both of these, so a blocking `systemctl restart` or
|
||
# `start` enqueues a job that systemd will not start until this
|
||
# unit is active — and this unit is not active until its ExecStart
|
||
# returns, which is waiting on that job. A deadlock, broken only
|
||
# when `TimeoutStartSec` fails the attempt, and entered again by
|
||
# every retry. Queueing the job and letting it run once we exit is
|
||
# the only ordering that terminates.
|
||
#
|
||
# The cost is real and is the reason `hive-tls-resign` does NOT
|
||
# do this: a propagation that fails after we return does not fail
|
||
# us, so the leaf can rotate on disk while nginx keeps the old
|
||
# copy. That unit blocks because nothing orders it before its
|
||
# consumers; this one cannot.
|
||
if systemctl is-active --quiet nginx.service; then
|
||
echo "swarm-services leaf rotated — re-importing and reloading nginx"
|
||
systemctl restart --no-block hive-gateway-self-signed-cert.service
|
||
systemctl reload --no-block nginx.service
|
||
elif [ "$(systemctl show -P LoadState nginx.service)" = loaded ]; then
|
||
echo "swarm-services leaf landed with nginx not running — starting it"
|
||
systemctl reset-failed nginx.service
|
||
systemctl start --no-block nginx.service
|
||
fi
|
||
|
||
# The bundle is assembled by `hive-tls-ca`, which runs BEFORE this
|
||
# unit — it has to, or the store's container cannot start and there
|
||
# is nothing to ask for a leaf. So on a first boot it wrote a bundle
|
||
# without the root below, and this reload is what completes it
|
||
# rather than a retry-path fallback. Only on a CHANGED root: every
|
||
# boot otherwise reruns a unit with nothing to do. A reload, not a
|
||
# restart: a restart restarts every unit that `Requires=` it.
|
||
if [ "$rootchanged" = 1 ] && systemctl is-active --quiet hive-tls-ca.service; then
|
||
echo "the services root changed — rebuilding the trust bundle"
|
||
systemctl reload --no-block hive-tls-ca.service
|
||
fi
|
||
'';
|
||
};
|
||
|
||
# The services leaf's renewal: `swarm-services-cert`'s own script, run
|
||
# again from the daily timer below.
|
||
#
|
||
# ⚠️ A unit of its own, not a timer on `swarm-services-cert`. That unit
|
||
# is `RemainAfterExit`, so once it has run it stays active and a timer
|
||
# starting it does nothing. And it cannot be restarted instead:
|
||
# `hive-gateway-self-signed-cert` `Requires=` it, so a restart restarts
|
||
# the gateway's cert import and nginx behind it, and a store that is
|
||
# sealed at that moment fails the start and takes both down over a leaf
|
||
# that was still valid. Nothing requires or orders against this unit, so
|
||
# its failure is its own.
|
||
#
|
||
# Failure leaves the leaf alone: every file the script writes is a
|
||
# `.new` until the store has answered with all three fields. No
|
||
# `Restart=`: the unit stays failed until the next tick, which is how
|
||
# a sealed store shows up in `systemctl --failed`.
|
||
systemd.services.swarm-services-cert-renew = {
|
||
description = "Renew the swarm-services TLS leaf from the secret store's PKI";
|
||
# The timer's `Persistent=` can fire it during boot. Behind the boot
|
||
# issuance, so it finds the leaf that run just wrote.
|
||
after = [ "swarm-services-cert.service" ];
|
||
path = servicesCertPath;
|
||
environment = servicesCertEnvironment;
|
||
inherit (config.systemd.services.swarm-services-cert) script;
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
UMask = "0077";
|
||
SyslogIdentifier = "swarm-services-cert-renew";
|
||
TimeoutStartSec = servicesCertTimeoutStartSec;
|
||
};
|
||
};
|
||
|
||
# Daily, for the reason `swarm-services-cert-renew` below is: the leaf
|
||
# needs the store, which can be sealed or unreachable.
|
||
systemd.timers.hive-tls-resign = lib.mkIf runsHive {
|
||
description = "Daily renewal check for the gateway TLS leaf";
|
||
wantedBy = [ "timers.target" ];
|
||
timerConfig = {
|
||
OnCalendar = "daily";
|
||
Persistent = true;
|
||
};
|
||
};
|
||
|
||
# Daily rather than weekly, for the reason
|
||
# ./swarm-nats.nix's `swarm-bao-nats-tls` timer is: this leaf needs the
|
||
# store, which can be sealed or unreachable, so there is a retry per day
|
||
# from half-life to expiry rather than two.
|
||
systemd.timers.swarm-services-cert-renew = {
|
||
description = "Daily renewal check for the swarm-services TLS leaf";
|
||
wantedBy = [ "timers.target" ];
|
||
timerConfig = {
|
||
OnCalendar = "daily";
|
||
Persistent = true;
|
||
};
|
||
};
|
||
})
|
||
];
|
||
}
|